Changeset 31

Timestamp:

06/30/09 13:17:30 (14 months ago)

Author:

fumanchu

Message:

Lots of changes, not all of them tested:

1. New --forward/--reverse options to commit command.
   
2. New --down option to diff command.
   
3. Previously broke update-to-a-revision by removing --revision option. Replaced it with a positional arg instead: "update DB [@REV]".
   
4. Added more whitelist info, including available type names, to pf usage output.
   
5. Removed now-unused --whitelist option.
   
6. New test and exception for whitelist values which are missing the required schema qualification.
   
7. Improved SQL comments in diff output declaring the url@rev and direction.
   
8. Improved the diff command to yield in order to have a chance at streaming its output.
   

Location:

postfacto/trunk/postfacto

Files:

• __init__.py (modified) (11 diffs)
• cmdline.py (modified) (8 diffs)
• differ.py (modified) (1 diff)
• whitelists.py (modified) (1 diff)

postfacto/trunk/postfacto/__init__.py

@@ -200 +200 @@
 
 def diff_plan(repo_db, relname, fromrev, torev):
-    """Yield (revision_id, diff) tuples to convert fromrev to torev.
+    """Yield (revision_id, diff, up?) tuples to convert fromrev to torev.
     
     The 'torev' argument MUST be an exact revision_id, and MUST be on the
@@ -217 +217 @@
         d, = repo_db.fetchone("SELECT %s FROM %s WHERE revision_id = %d;" %
                               (field, relname, revid))
-        yield revid, d
+        yield revid, d, field == 'diffup'
 
 
@@ -340 +340 @@
             wr=wrev, wc=wcopy, s=local_postfacto) as txmgr:
         # Apply the diffs to our working revision...
-        for revid, d in plan:
+        for revid, d, up in plan:
             if d:
                 try:
@@ -349 +349 @@
         
         # ...and apply the same diffs to our working copy.
-        for revid, d in plan:
+        for revid, d, up in plan:
             if d:
                 try:
@@ -449 +449 @@
 
 
-def update(working_copy, revision=None, force=False):
+def update(working_copy, revision=None):
     """Update the working copy to the given revision. Return the revision id."""
     co = info(working_copy)
@@ -551 +551 @@
 
 
-def diff(source, target):
+def diff(source, target, down=False):
     """Return a diff between the given source and target.
     
@@ -572 +572 @@
             if ':' in revision:
                 fromrev, torev = revision.split(':', 1)
+                if down:
+                    fromrev, torev = torev, fromrev
                 fromrev = closest_revision(repo_db, relname, branch, fromrev)
                 torev = closest_revision(repo_db, relname, branch, torev)
                 plan = diff_plan(repo_db, relname, fromrev, torev)
-                return [('-- Postfacto revision: %s@%d --\n' %
-                         (baseurl, revid)) + d
-                        for revid, d in plan]
+                for revid, d, up in plan:
+                    yield ('-- Postfacto revision: %s@%d (%s)--' %
+                           (baseurl, revid, {True: 'up', False: 'down'}[up]))
+                    for chunk in d.split('\n\n\n'):
+                        yield chunk
+                return
             else:
                 rev = closest_revision(repo_db, relname, branch, revision)
-                diffup, = repo_db.fetchone(
-                    "SELECT diffup FROM %s WHERE revision_id = %d" % (relname, rev))
-                return ['-- Postfacto revision: %s@%d --\n' %
-                        (baseurl, rev)] + diffup.split('\n\n\n')
+                if down:
+                    col = 'down'
+                else:
+                    col = 'up'
+                d, = repo_db.fetchone(
+                    "SELECT diff%s FROM %s WHERE revision_id = %d" %
+                    (col, relname, rev))
+                yield ('-- Postfacto revision: %s@%d (%s) --\n' %
+                       (baseurl, rev, col))
+                for chunk in d.split('\n\n\n'):
+                    yield chunk
+                return
         else:
             # target must be a working_copy DB.
@@ -590 +603 @@
             if not db1.exists():
                 raise Failure("The working revision database does not exist.")
+            
             db2 = postgres.Database(target, local_server)
             if not db2.exists():
@@ -600 +614 @@
             wl2 = wl1.union(Whitelist.from_str(co['add']))
             wl2 = wl2.difference(Whitelist.from_str(co['delete']))
+            
+            prologue = ('-- Postfacto working copy diff: %s (%s)--' %
+                        (target, {True: 'up', False: 'down'}[down]))
     else:
         raise NotImplemented
     
     d = differ.Differ(db1, db2, wl1, wl2)
-    forward = list(d.forward_diff())
-    if forward == [d.param_block]:
-        return []
-    return forward
-
-
-def commit(working_copy, message, force=False):
-    """Commit changes in working copy to repo. Return new revision number."""
+    if down:
+        diff = d.reverse_diff()
+    else:
+        diff = d.forward_diff()
+    param_block = diff.next()
+    if param_block == d.param_block:
+        # If this next line raises StopIteration, there are no changes,
+        # so we won't yield anything if we propagate it outward.
+        first_chunk = diff.next()
+        yield prologue
+        yield param_block
+        yield first_chunk
+    else:
+        yield prologue
+        yield param_block
+    
+    for chunk in diff:
+        yield chunk
+
+def commit(working_copy, message, force=False, forward=None, reverse=None):
+    """Commit changes in working copy to repo. Return new revision number.
+    
+    If 'forward' is given, it will be inserted as the diffup value instead
+    of calculating the difference between the working copy and its base.
+    
+    If 'reverse' is given, it will be inserted as the diffdown value instead
+    of calculating the difference between the working copy and its base.
+    
+    Both of these allow a diff to include RENAMEs, INSERTs, UPDATEs,
+    etc. which are not able to be inferred from the system catalog.
+    Use 'pf diff DB > up.sql' and 'pf diff --reverse DB > down.sql'
+    to obtain files which you can then modify and commit using
+    'commit --forward=up.sql --reverse=down.sql DB'.
+    
+    Note that the whitelists for the working copy will still be used even if
+    --forward and --reverse are provided; that is, you must still use the
+    'add' and 'remove' commands in sync with the provided diffs.
+    """
     co = info(working_copy)
     wl = Whitelist.from_str(co['whitelist'])
+    # First, expand the whitelist to include any added objects...
+    wl2 = wl.union(Whitelist.from_str(co['add']))
+    # ...and subtract any deleted objects.
+    wl2 = wl2.difference(Whitelist.from_str(co['delete']))
     
     server = server_from_netloc(co['server'])
@@ -620 +671 @@
         raise Failure("The repository database %r does not exist." % co['repo'])
     
-    # Get our diffs.
     wrev = postgres.Database('postfacto_w_%s' % working_copy, local_server)
     if not wrev.exists():
@@ -627 +677 @@
     if not wcopy.exists():
         raise Failure("The given working copy %r does not exist." % working_copy)
-    # We must first expand the whitelist to include any added objects.
-    wl2 = wl.union(Whitelist.from_str(co['add']))
-    # ...and now we merge our deleted list
-    wl2 = wl2.difference(Whitelist.from_str(co['delete']))
-    d = differ.Differ(wrev, wcopy, wl, wl2)
-    forward = list(d.forward_diff())
-    reverse = list(d.reverse_diff())
+    
+    # Get our diffs.
+    if not (forward and reverse):
+        d = differ.Differ(wrev, wcopy, wl, wl2)
+    if forward:
+        forward = open(forward, 'rb').read()
+    else:
+        forward = list(d.forward_diff())
+    if reverse:
+        reverse = open(reverse, 'rb').read()
+    else:
+        reverse = list(d.reverse_diff())
     
     if forward == [d.param_block] and reverse == [d.param_block] and not force:

postfacto/trunk/postfacto/cmdline.py

@@ -17 +17 @@
                 'add [--no-expand] DB whitelist',
                 'delete [--no-expand] DB whitelist',
-                'commit --message [--force] DB',
+                'commit --message [--force] [--forward] [--reverse] DB',
                 'copy [--message] URL branch',
-                'diff URL|DB',
+                'diff [--down] URL|DB',
                 'info DB',
                 'import [--message] [--force] [--no-checkout] DB URL',
@@ -27 +27 @@
                 'revert DB',
                 'switch URL DB',
-                'update DB',
+                'update DB [REV]',
                 ]
     
@@ -33 +33 @@
         OptionParser.__init__(self)
         self.usage = '%prog [options] command *args'
+        wltypes = sorted(postfacto.differ.whitelist_types)
         self.epilog = (
             os.linesep +
             'URLs: postfacto://[[user@]server]/repodb[/project[/branch][@revision]]' +
+            os.linesep +
+            'Whitelist example: tables=users,accounts;schemas=test' +
+            os.linesep +
+            'Whitelist types  : ' + ', '.join(wltypes) +
             os.linesep +
             'Commands:' +
@@ -60 +65 @@
                         action='store_true', dest='force',
                         help="Ignore warnings when performing the operation.")
-        self.add_option('-w', '--whitelist', default=None,
-                        action='store', type='string', dest='whitelist',
-                        help="Whitelist string: cat1=name1,name2;cat2=name3")
         self.add_option('--no-expand', default=False,
                         action='store_true', dest='no_expand',
@@ -71 +73 @@
                         help="Do not register the imported DB as a checked "
                              "out copy.")
+        self.add_option('--forward', default=None,
+                        action='store', dest='forward',
+                        help="File to store instead of working copy diff up.")
+        self.add_option('--reverse', default=None,
+                        action='store', dest='reverse',
+                        help="File to store instead of working copy diff down.")
+        self.add_option('--down', default=False,
+                        action='store_true', dest='down',
+                        help="Return a reverse diff instead of a forward diff.")
     
     def format_epilog(self, formatter):
@@ -113 +124 @@
                 raise postfacto.Failure(
                     'You must supply a working copy name and whitelist.')
-            wl = postfacto.add(args[0], args[1].strip(),
-                               expand=(not opts.no_expand))
+            try:
+                wl = postfacto.add(args[0], args[1].strip(),
+                                   expand=(not opts.no_expand))
+            except postfacto.whitelists.WhitelistError, x:
+                print x
+                sys.exit(1)
             print "Added %d objects:" % sum([len(v) for v in wl.values()], 0)
             for k in sorted(wl.keys()):
@@ -141 +156 @@
                 raise postfacto.Failure(
                     'You must supply a working copy name and a --message.')
-            rev = postfacto.commit(args[0], opts.message, force=opts.force)
+            rev = postfacto.commit(args[0], opts.message, force=opts.force,
+                                   forward=opts.forward, reverse=opts.reverse)
             print "Committed revision %d." % rev
         elif command == 'diff':
             started = False
             if len(args) == 1:
-                for block in postfacto.diff(source=None, target=args[0]):
+                for block in postfacto.diff(source=None, target=args[0],
+                                            down=opts.down):
                     if not started:
                         started = True
@@ -213 +230 @@
             print "Updated to revision %d." % rev
         elif command == 'update':
-            rev = postfacto.update(args[0])
+            revision = None
+            if len(args) == 2:
+                revision = args[1]
+            rev = postfacto.update(args[0], revision=revision)
             print "Updated to revision %d." % rev
     except postfacto.Failure, f:

postfacto/trunk/postfacto/differ.py

@@ -2092 +2092 @@
             raise ValueError("Circular dependencies found.", remaining)
 
+
+whitelist_types = set()
+schema_whitelist_types = set()
+for v in vars().values():
+    if isinstance(v, type) and issubclass(v, DumpableObject):
+        t = v.whitelist_type
+        if t:
+            whitelist_types.add(t)
+            if issubclass(v, SchemaObject):
+                schema_whitelist_types.add(t)
+        del t
+del v
+

postfacto/trunk/postfacto/whitelists.py

@@ -12 +12 @@
 
 
+class WhitelistError(Exception):
+    pass
+
 class Whitelist(dict):
     """A dict of {category name: set(object names)} pairs."""
+    
+    def __setitem__(self, k, v):
+        if k in postfacto.differ.schema_whitelist_types:
+            for i in v:
+                if '.' not in i:
+                    raise WhitelistError("The given value '%s=%s' is not "
+                                         "schema-qualified." % (k, i))
     
     def __str__(self):