Synchronize with batch size

[Allan Frank-2]

Hi

Sorry, here's the patch.

/Allan

diff -u -r src.orig/org/jets3t/apps/synchronize/Synchronize.java src/org/jets3t/apps/synchronize/Synchronize.java
--- src.orig/org/jets3t/apps/synchronize/Synchronize.java 2008-08-03 21:44:28.000000000 +0200
+++ src/org/jets3t/apps/synchronize/Synchronize.java 2008-12-07 19:14:23.000000000 +0100
@@ -30,6 +30,8 @@
 import java.util.Map;
 import java.util.Set;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.jets3t.service.Constants;
 import org.jets3t.service.Jets3tProperties;
 import org.jets3t.service.S3Service;
@@ -65,6 +67,7 @@
  * @author James Murty
  */
 public class Synchronize {
+    private static final Log log = LogFactory.getLog(Synchronize.class);
     public static final String APPLICATION_DESCRIPTION = "Synchronize/0.6.1";
     
     protected static final int REPORT_LEVEL_NONE = 0;
@@ -155,7 +158,21 @@
         this.properties = properties;
         this.fileComparer = FileComparer.getInstance(properties);
     }
-    
+    class ObjectToBePrepared {
+     String targetKey;
+     File file;
+     String aclString;
+     EncryptionUtil encryptionUtil;
+     public ObjectToBePrepared( String targetKey, File file, String aclString, EncryptionUtil encryptionUtil ) {
+     this.targetKey = targetKey;
+     this.file = file;
+     this.aclString = aclString;
+     this.encryptionUtil = encryptionUtil;
+     }
+     private S3Object prepareThisUploadObject() throws Exception {
+     return prepareUploadObject(targetKey, file, aclString, encryptionUtil);
+     }
+    }
 
     /**
      * Prepares a file to be uploaded to S3, creating an S3Object with the
@@ -322,7 +339,8 @@
             ArrayList sortedFilesKeys = new ArrayList(filesMap.keySet());
             Collections.sort(sortedFilesKeys);
             
-            Set objectsToUpload = new HashSet();
+            //ArrayList<ObjectToBePrepared> objectsToUpload = new ArrayList<ObjectToBePrepared>();
+            ArrayList objectsToUpload = new ArrayList();
             
             // Iterate through local files and perform the necessary action to synchronise them with S3.
             Iterator fileKeyIter = sortedFilesKeys.iterator();
@@ -356,14 +374,14 @@
     
                 if (discrepancyResults.onlyOnClientKeys.contains(relativeKeyPath)) {
                     printOutputLine("N " + relativeKeyPath, REPORT_LEVEL_ACTIONS);
-                    objectsToUpload.add(prepareUploadObject(targetKey, file, aclString, encryptionUtil));
+                    objectsToUpload.add(new ObjectToBePrepared(targetKey, file, aclString, encryptionUtil));
                 } else if (discrepancyResults.updatedOnClientKeys.contains(relativeKeyPath)) {
                     printOutputLine("U " + relativeKeyPath, REPORT_LEVEL_ACTIONS);
-                    objectsToUpload.add(prepareUploadObject(targetKey, file, aclString, encryptionUtil));
+                    objectsToUpload.add(new ObjectToBePrepared(targetKey, file, aclString, encryptionUtil));
                 } else if (discrepancyResults.alreadySynchronisedKeys.contains(relativeKeyPath)) {
                     if (isForce) {
                         printOutputLine("F " + relativeKeyPath, REPORT_LEVEL_ACTIONS);
-                        objectsToUpload.add(prepareUploadObject(targetKey, file, aclString, encryptionUtil));
+                        objectsToUpload.add(new ObjectToBePrepared(targetKey, file, aclString, encryptionUtil));
                     } else {
                         printOutputLine("- " + relativeKeyPath, REPORT_LEVEL_ALL);
                     }
@@ -373,7 +391,7 @@
                         printOutputLine("r " + relativeKeyPath, REPORT_LEVEL_DIFFERENCES);                    
                     } else {
                         printOutputLine("R " + relativeKeyPath, REPORT_LEVEL_ACTIONS);
-                        objectsToUpload.add(prepareUploadObject(targetKey, file, aclString, encryptionUtil));
+                        objectsToUpload.add(new ObjectToBePrepared(targetKey, file, aclString, encryptionUtil));
                     }
                 } else {
                     // Uh oh, program error here. The safest thing to do is abort!
@@ -384,8 +402,19 @@
             }
                     
             // Upload New/Updated/Forced/Replaced objects to S3.
-            if (doAction && objectsToUpload.size() > 0) {
-                S3Object[] objects = (S3Object[]) objectsToUpload.toArray(new S3Object[objectsToUpload.size()]);
+            while (doAction && objectsToUpload.size() > 0) {
+                int uploadbatchsize = properties.getIntProperty("upload.batchsize", 0); //user supplied batch size
+                log.debug("upload.batchsize="+uploadbatchsize);
+                if( uploadbatchsize == 0 || uploadbatchsize > objectsToUpload.size() ) { //if not supplied or larger than all objects, set to all objects
+                    uploadbatchsize = objectsToUpload.size();
+                }
+                log.debug("uploadbatchsize="+uploadbatchsize);
+                //S3Object[] objects = (S3Object[]) objectsToUpload.toArray(new S3Object[objectsToUpload.size()]);
+                S3Object[] objects = new S3Object[uploadbatchsize];
+                for(int i=0;i<uploadbatchsize;i++) {
+                 ObjectToBePrepared obj = (ObjectToBePrepared)objectsToUpload.remove(0);
+                 objects[i] = obj.prepareThisUploadObject();
+                }
                 (new S3ServiceMulti(s3Service, serviceEventAdaptor)).putObjects(bucket, objects);
                 if (serviceEventAdaptor.wasErrorThrown()) {
                     Throwable thrown = serviceEventAdaptor.getErrorThrown();


---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@...
For additional commands, e-mail: dev-help@...