From 1288d43585e5347b737ef9f226f34c22d64db960 Mon Sep 17 00:00:00 2001 From: Danila Shvets Date: Tue, 21 May 2024 14:56:16 +0300 Subject: [PATCH] Add support for "uuid" type in tracking_column_type * Added "uuid" type to tracking_column_type settings * Updated relevant documentation * Added tests for "uuid" type * Ensured compatibility with existing types --- docs/input-jdbc.asciidoc | 24 ++++++++++--- lib/logstash/inputs/jdbc.rb | 4 +-- .../plugin_mixins/jdbc/value_tracking.rb | 36 +++++++++++++------ spec/inputs/jdbc_spec.rb | 8 +++-- 4 files changed, 52 insertions(+), 20 deletions(-) diff --git a/docs/input-jdbc.asciidoc b/docs/input-jdbc.asciidoc index 2423217..7950814 100644 --- a/docs/input-jdbc.asciidoc +++ b/docs/input-jdbc.asciidoc @@ -127,14 +127,14 @@ Here is the list: |========================================================== |sql_last_value | The value used to calculate which rows to query. Before any query is run, -this is set to Thursday, 1 January 1970, or 0 if `use_column_value` is true and +this is set to Thursday, 1 January 1970, 0, or 00000000-0000-0000-0000-000000000000 if `use_column_value` is true and `tracking_column` is set. It is updated accordingly after subsequent queries are run. |offset, size| Values used with manual paging mode to explicitly implement the paging. Supported only if <> is enabled and <> has the `explicit` value. |========================================================== -Example: +Examples: [source,ruby] --------------------------------------------------------------------------------------------------- input { @@ -240,7 +240,7 @@ This plugin supports the following configuration options plus the <> |a valid filesystem path|No | <> | {logstash-ref}/field-references-deepdive.html[field reference] | No | <> |<>|No -| <> |<>, one of `["numeric", "timestamp"]`|No +| <> |<>, one of `["numeric", "timestamp", "uuid"]`|No | <> |<>|No | <> |<>|No |======================================================================= @@ -645,10 +645,24 @@ The column whose value is to be tracked if `use_column_value` is set to `true` [id="plugins-{type}s-{plugin}-tracking_column_type"] ===== `tracking_column_type` - * Value can be any of: `numeric`, `timestamp` + * Value can be any of: `numeric`, `timestamp`, `uuid` * Default value is `"numeric"` -Type of tracking column. Currently only "numeric" and "timestamp" +Type of tracking column. Currently only "numeric", "timestamp" and "uuid" + +NOTE: Example of uuid type usage in PostgreSQL. +[source,ruby] +--------------------------------------------------------------------------------------------------- +input { + jdbc { + statement => "SELECT id, mycolumn1, mycolumn2 FROM my_table WHERE id > :sql_last_value::uuid" + use_column_value => true + tracking_column => "id" + tracking_column_type => "uuid" + # ... other configuration bits + } +} +--------------------------------------------------------------------------------------------------- [id="plugins-{type}s-{plugin}-use_column_value"] ===== `use_column_value` diff --git a/lib/logstash/inputs/jdbc.rb b/lib/logstash/inputs/jdbc.rb index 9c9dadb..6754cda 100755 --- a/lib/logstash/inputs/jdbc.rb +++ b/lib/logstash/inputs/jdbc.rb @@ -191,8 +191,8 @@ module LogStash module Inputs class Jdbc < LogStash::Inputs::Base # If tracking column value rather than timestamp, the column whose value is to be tracked config :tracking_column, :validate => :string - # Type of tracking column. Currently only "numeric" and "timestamp" - config :tracking_column_type, :validate => ['numeric', 'timestamp'], :default => 'numeric' + # Type of tracking column. Currently only "numeric", "timestamp" and "uuid" + config :tracking_column_type, :validate => ['numeric', 'timestamp', 'uuid'], :default => 'numeric' # Whether the previous run state should be preserved config :clean_run, :validate => :boolean, :default => false diff --git a/lib/logstash/plugin_mixins/jdbc/value_tracking.rb b/lib/logstash/plugin_mixins/jdbc/value_tracking.rb index a570ea4..65be948 100644 --- a/lib/logstash/plugin_mixins/jdbc/value_tracking.rb +++ b/lib/logstash/plugin_mixins/jdbc/value_tracking.rb @@ -12,16 +12,21 @@ def self.build_last_value_tracker(plugin) handler.clean end - if plugin.use_column_value && plugin.tracking_column_type == "numeric" - # use this irrespective of the jdbc_default_timezone setting - NumericValueTracker.new(handler) - else - if plugin.jdbc_default_timezone.nil? - # no TZ stuff for Sequel, use Time - TimeValueTracker.new(handler) + if plugin.use_column_value + case plugin.tracking_column_type + when "numeric" + # use this irrespective of the jdbc_default_timezone setting + NumericValueTracker.new(handler) + when "uuid" + UuidValueTracker.new(handler) else - # Sequel does timezone handling on DateTime only - DateTimeValueTracker.new(handler) + if plugin.jdbc_default_timezone.nil? + # no TZ stuff for Sequel, use Time + TimeValueTracker.new(handler) + else + # Sequel does timezone handling on DateTime only + DateTimeValueTracker.new(handler) + end end end end @@ -34,7 +39,7 @@ def initialize(handler) end if Psych::VERSION&.split('.')&.first.to_i >= 4 - YAML_PERMITTED_CLASSES = [::DateTime, ::Time, ::BigDecimal].freeze + YAML_PERMITTED_CLASSES = [::DateTime, ::Time, ::BigDecimal, ::String].freeze def self.load_yaml(source) Psych::safe_load(source, permitted_classes: YAML_PERMITTED_CLASSES) end @@ -81,6 +86,17 @@ def set_value(value) end end + class UuidValueTracker < ValueTracking + def set_initial + common_set_initial(:uuid, "00000000-0000-0000-0000-000000000000") + end + + def set_value(value) + return unless value.is_a?(String) + @value = value + end + end + class DateTimeValueTracker < ValueTracking def set_initial common_set_initial(:to_datetime, DateTime.new(1970)) diff --git a/spec/inputs/jdbc_spec.rb b/spec/inputs/jdbc_spec.rb index a9fa8db..f9ac3f0 100755 --- a/spec/inputs/jdbc_spec.rb +++ b/spec/inputs/jdbc_spec.rb @@ -35,11 +35,12 @@ db.create_table :test_table do DateTime :created_at BigDecimal :big_num + String :uuid Integer :num String :string DateTime :custom_time end - db << "CREATE TABLE types_table (num INTEGER, string VARCHAR(255), started_at DATE, custom_time TIMESTAMP, ranking DECIMAL(16,6))" + db << "CREATE TABLE types_table (num INTEGER, string VARCHAR(255), started_at DATE, custom_time TIMESTAMP, ranking DECIMAL(16,6), uuid VARCHAR(36))" db << "CREATE TABLE test1_table (num INTEGER, string VARCHAR(255), custom_time TIMESTAMP, created_at TIMESTAMP)" end end @@ -1522,7 +1523,7 @@ end before do - db << "INSERT INTO types_table (num, string, started_at, custom_time, ranking) VALUES (1, 'A test', '1999-12-31', '1999-12-31 23:59:59', 95.67)" + db << "INSERT INTO types_table (num, string, started_at, custom_time, ranking, uuid) VALUES (1, 'A test', '1999-12-31', '1999-12-31 23:59:59', 95.67, '018f15f3-6cfd-7a1b-b70f-d97ed8a73128')" plugin.register end @@ -1539,6 +1540,7 @@ expect(event.get("started_at")).to be_a_logstash_timestamp_equivalent_to("1999-12-31T00:00:00.000Z") expect(event.get("custom_time")).to be_a_logstash_timestamp_equivalent_to("1999-12-31T23:59:59.000Z") expect(event.get("ranking").to_f).to eq(95.67) + expect(event.get("uuid")).to eq("018f15f3-6cfd-7a1b-b70f-d97ed8a73128") end end @@ -1552,7 +1554,7 @@ end before(:each) do - db << "INSERT INTO types_table (num, string, started_at, custom_time, ranking) VALUES (1, 'A test', '1999-12-31', '2021-11-07 01:23:45', 95.67)" + db << "INSERT INTO types_table (num, string, started_at, custom_time, ranking, uuid) VALUES (1, 'A test', '1999-12-31', '2021-11-07 01:23:45', 95.67, '018f15f3-6cfd-7a1b-b70f-d97ed8a73128')" plugin.register end