Add support for detecting children moving to different parents

Author: agardiner

lib/csv-diff/algorithm.rb

@@ -3,6 +3,55 @@ class CSVDiff
     # Implements the CSV diff algorithm.
     module Algorithm
 
+        # Holds the details of a single difference
+        class Diff
+
+            attr_accessor :diff_type
+            attr_reader :fields
+            attr_reader :row
+            attr_reader :sibling_position
+
+            def initialize(diff_type, fields, row_idx, pos_idx)
+                @diff_type = diff_type
+                @fields = fields
+                @row = row_idx + 1
+                self.sibling_position = pos_idx
+            end
+
+
+            def sibling_position=(pos_idx)
+                if pos_idx.is_a?(Array)
+                    pos_idx.compact!
+                    if pos_idx.first != pos_idx.last
+                        @sibling_position = pos_idx.map{ |pos| pos + 1 }
+                    else
+                        @sibling_position = pos_idx.first + 1
+                    end
+                else
+                    @sibling_position = pos_idx + 1
+                end
+            end
+
+
+            # For backwards compatibility and access to fields with differences
+            def [](key)
+                case key
+                when String
+                    @fields[key]
+                when :action
+                    a = diff_type.to_s
+                    a[0] = a[0].upcase
+                    a
+                when :row
+                    @row
+                when :sibling_position
+                    @sibling_position
+                end
+            end
+
+        end
+
+
         # Diffs two CSVSource structures.
         #
         # @param left [CSVSource] A CSVSource object containing the contents of
@@ -33,74 +82,83 @@ def diff_sources(left, right, key_fields, diff_fields, options = {})
             right_index = right.index
             right_values = right.lines
             right_keys = right_values.keys
-            parent_fields = left.parent_fields.length
+            parent_field_count = left.parent_fields.length
 
             include_adds = !options[:ignore_adds]
             include_moves = !options[:ignore_moves]
             include_updates = !options[:ignore_updates]
             include_deletes = !options[:ignore_deletes]
 
-            diffs = Hash.new{ |h, k| h[k] = {} }
+            diffs = {}
+            potential_moves = Hash.new{ |h, k| h[k] = [] }
 
             # First identify deletions
             if include_deletes
                 (left_keys - right_keys).each do |key|
                     # Delete
                     key_vals = key.split('~', -1)
-                    parent = key_vals[0...parent_fields].join('~')
+                    parent = key_vals[0...parent_field_count].join('~')
+                    child = key_vals[parent_field_count..-1].join('~')
                     left_parent = left_index[parent]
                     left_value = left_values[key]
-                    left_idx = left_parent.index(key)
-                    next unless left_idx
-                    id = {}
-                    id[:row] = left_keys.index(key) + 1
-                    id[:sibling_position] = left_idx + 1
-                    key_fields.each do |field_name|
-                        id[field_name] = left_value[field_name]
-                    end
-                    diffs[key].merge!(id.merge(left_values[key].merge(:action => 'Delete')))
+                    row_idx = left_keys.index(key)
+                    sib_idx = left_parent.index(key)
+                    raise "Can't locate key #{key} in parent #{parent}" unless sib_idx
+                    diffs[key] = d = Diff.new(:delete, left_value, row_idx, sib_idx)
+                    potential_moves[child] << key
                     #puts "Delete: #{key}"
                 end
             end
 
             # Now identify adds/updates
             right_keys.each_with_index do |key, right_row_id|
                 key_vals = key.split('~', -1)
-                parent = key_vals[0...parent_fields].join('~')
+                parent = key_vals[0...parent_field_count].join('~')
                 left_parent = left_index[parent]
                 right_parent = right_index[parent]
                 left_value = left_values[key]
                 right_value = right_values[key]
                 left_idx = left_parent && left_parent.index(key)
                 right_idx = right_parent && right_parent.index(key)
 
-                id = {}
-                id[:row] = right_row_id + 1
-                id[:sibling_position] = right_idx + 1
-                key_fields.each do |field_name|
-                    id[field_name] = right_value[field_name]
-                end
                 if left_idx && right_idx
+                    if include_updates && (changes = diff_row(left_value, right_value, diff_fields, case_sensitive))
+                        id = id_fields(key_fields, right_value)
+                        diffs[key] = Diff.new(:update, id.merge!(changes), right_row_id, right_idx)
+                        #puts "Change: #{key}"
+                    end
                     if include_moves
                         left_common = left_parent & right_parent
                         right_common = right_parent & left_parent
                         left_pos = left_common.index(key)
                         right_pos = right_common.index(key)
                         if left_pos != right_pos
                             # Move
-                            diffs[key].merge!(id.merge!(:action => 'Move',
-                                              :sibling_position => [left_idx + 1, right_idx + 1]))
+                            if d = diffs[key]
+                                d.sibling_position = [left_idx, right_idx]
+                            else
+                                id = id_fields(key_fields, right_value)
+                                diffs[key] = Diff.new(:move, id, right_row_id, [left_idx, right_idx])
+                            end
                             #puts "Move #{left_idx} -> #{right_idx}: #{key}"
                         end
                     end
-                    if include_updates && (changes = diff_row(left_value, right_value, diff_fields, case_sensitive))
-                        diffs[key].merge!(id.merge(changes.merge(:action => 'Update')))
-                        #puts "Change: #{key}"
-                    end
-                elsif include_adds && right_idx
+                elsif right_idx
                     # Add
-                    diffs[key].merge!(id.merge(right_values[key].merge(:action => 'Add')))
-                    #puts "Add: #{key}"
+                    child = key_vals[parent_field_count..-1].join('~')
+                    if potential_moves.has_key?(child) && old_key = potential_moves[child].pop
+                        diffs.delete(old_key)
+                        if include_updates
+                            left_value = left_values[old_key]
+                            id = id_fields(right.child_fields, right_value)
+                            changes = diff_row(left_value, right_value, left.parent_fields + diff_fields, case_sensitive)
+                            diffs[key] = Diff.new(:update, id.merge!(changes), right_row_id, right_idx)
+                            #puts "Update Parent: #{key}"
+                        end
+                    elsif include_adds
+                        diffs[key] = Diff.new(:add, right_value, right_row_id, right_idx)
+                        #puts "Add: #{key}"
+                    end
                 end
             end
 
@@ -137,6 +195,19 @@ def diff_row(left_row, right_row, fields, case_sensitive)
             diffs if diffs.size > 0
         end
 
+
+        private
+
+
+        # Return a hash containing just the key field values
+        def id_fields(key_fields, fields)
+            id = {}
+            key_fields.each do |field_name|
+                id[field_name] = fields[field_name]
+            end
+            id
+        end
+
     end
 
 end

lib/csv-diff/csv_source.rb

@@ -6,20 +6,29 @@ class CSVSource
 
         # @return [String] the path to the source file
         attr_accessor :path
+
         # @return [Array<String>] The names of the fields in the source file
         attr_reader :field_names
         # @return [Array<String>] The names of the field(s) that uniquely
         #   identify each row.
         attr_reader :key_fields
-        # @return [Array<Fixnum>] The indexes of the key fields in the source
-        #   file.
-        attr_reader :key_field_indexes
         # @return [Array<String>] The names of the field(s) that identify a
         #   common parent of child records.
         attr_reader :parent_fields
         # @return [Array<String>] The names of the field(s) that distinguish a
         #   child of a parent record.
         attr_reader :child_fields
+
+        # @return [Array<Fixnum>] The indexes of the key fields in the source
+        #   file.
+        attr_reader :key_field_indexes
+        # @return [Array<Fixnum>] The indexes of the parent fields in the source
+        #   file.
+        attr_reader :parent_field_indexes
+        # @return [Array<Fixnum>] The indexes of the child fields in the source
+        #   file.
+        attr_reader :child_field_indexes
+
         # @return [Boolean] True if the source has been indexed with case-
         #   sensitive keys, or false if it has been indexed using upper-case key
         #   values.
@@ -117,7 +126,9 @@ def [](key)
         def index_source(lines, options)
             @lines = {}
             @index = Hash.new{ |h, k| h[k] = [] }
-            @key_field_indexes = find_field_indexes(@key_fields, @field_names) if @field_names
+            if @field_names
+                index_fields
+            end
             @case_sensitive = options.fetch(:case_sensitive, true)
             @trim_whitespace = options.fetch(:trim_whitespace, false)
             line_num = 0
@@ -126,8 +137,7 @@ def index_source(lines, options)
                 next if line_num == 1 && @field_names && options[:ignore_header]
                 unless @field_names
                     @field_names = row
-                    @key_field_indexes = find_field_indexes(@key_fields, @field_names)
-                    @key_fields = @key_field_indexes.map{ |i| @field_names[i] }
+                    index_fields
                     next
                 end
                 field_vals = row
@@ -150,6 +160,16 @@ def index_source(lines, options)
         end
 
 
+        def index_fields
+            @key_field_indexes = find_field_indexes(@key_fields, @field_names)
+            @parent_field_indexes = find_field_indexes(@parent_fields, @field_names)
+            @child_field_indexes = find_field_indexes(@child_fields, @field_names)
+            @key_fields = @key_field_indexes.map{ |i| @field_names[i] }
+            @parent_fields = @parent_field_indexes.map{ |i| @field_names[i] }
+            @child_fields = @child_field_indexes.map{ |i| @field_names[i] }
+        end
+
+
         # Converts an array of field names to an array of indexes of the fields
         # matching those names.
         def find_field_indexes(key_fields, field_names)

test/test_diff.rb

@@ -9,30 +9,37 @@ class TestDiff < Test::Unit::TestCase
         ['A', 'A1', 'Account1'],
         ['A', 'A2', 'Account 2'],
         ['A', 'A3', 'Account 3'],
-        ['A', 'A4', 'Account 4']
+        ['A', 'A4', 'Account 4'],
+        ['A', 'A6', 'Account 6']
     ]
 
     DATA2 = [
         ['Parent', 'Child', 'Description'],
         ['A', 'A1', 'Account1'],
         ['A', 'A2', 'Account2'],
         ['A', 'a3', 'ACCOUNT 3'],
-        ['A', 'A5', 'Account 5']
+        ['A', 'A5', 'Account 5'],
+        ['B', 'A6', 'Account 6'],
+        ['C', 'A6', 'Account 6c']
     ]
 
     def test_array_diff
-        diff = CSVDiff.new(DATA1, DATA2, key_fields: [1, 0])
-        assert_equal(1, diff.adds.size)
+        diff = CSVDiff.new(DATA1, DATA2, key_fields: [0, 1])
+        #assert_equal(['Parent'], diff.left.parent_fields)
+        #assert_equal(['Parent'], diff.right.parent_fields)
+        #assert_equal(['Child'], diff.left.child_fields)
+        #assert_equal(['Child'], diff.right.child_fields)
+        assert_equal(2, diff.adds.size)
         assert_equal(1, diff.deletes.size)
-        assert_equal(2, diff.updates.size)
+        assert_equal(3, diff.updates.size)
     end
 
 
     def test_case_insensitive_diff
-        diff = CSVDiff.new(DATA1, DATA2, key_fields: [1, 0], case_sensitive: false)
-        assert_equal(1, diff.adds.size)
+        diff = CSVDiff.new(DATA1, DATA2, key_fields: [0, 1], case_sensitive: false)
+        assert_equal(2, diff.adds.size)
         assert_equal(1, diff.deletes.size)
-        assert_equal(1, diff.updates.size)
+        assert_equal(2, diff.updates.size)
     end