[gatewiki-commits] SF.net SVN: gatewiki:[1787] trunk/cow
|
View:
New views
1 Messages
—
Rating Filter:
Alert me
|
 |
[gatewiki-commits] SF.net SVN: gatewiki:[1787] trunk/cow
by ian_roberts
::
Rate this Message:
Revision: 1787
http://gatewiki.svn.sourceforge.net/gatewiki/?rev=1787&view=rev Author: ian_roberts Date: 2009-11-04 16:42:41 +0000 (Wed, 04 Nov 2009) Log Message: ----------- Working on upload. Now handles zip files with directory entries correctly, also handles tar(.gz|.bz2) archives. Need to add more tests for this, but the next task is to move the uploads to use the staging area rather than uploading directly into the sandbox. Modified Paths: -------------- trunk/cow/grails-app/controllers/PageController.groovy trunk/cow/grails-app/services/PageService.groovy trunk/cow/selenium/tests/upload-server.html trunk/cow/selenium/tests/upload.html trunk/cow/src/java/gate/cow/gwt/client/Upload.java trunk/cow/src/java/gate/versioning/svnkit/Sandbox.java Modified: trunk/cow/grails-app/controllers/PageController.groovy =================================================================== --- trunk/cow/grails-app/controllers/PageController.groovy 2009-11-03 16:04:13 UTC (rev 1786) +++ trunk/cow/grails-app/controllers/PageController.groovy 2009-11-04 16:42:41 UTC (rev 1787) @@ -12,6 +12,7 @@ import org.springframework.util.StringUtils import org.springframework.web.multipart.MultipartHttpServletRequest import org.springframework.web.multipart.commons.CommonsMultipartFile +import org.tmatesoft.svn.core.wc.SVNStatusType import gate.cow.CowUtils import gate.util.FileUtils import gate.versioning.svnkit.Sandbox @@ -189,151 +190,94 @@ log.debug("upload, params = ${params},\n thisPage=${thisPage}") Wiki wiki = thisPage.wiki // get the uploaded file from <input type='file' name='fileUpload'> - assert request instanceof MultipartHttpServletRequest - def file = request.getFile('fileUpload') as CommonsMultipartFile + def multipartRequest + if(request instanceof MultipartHttpServletRequest) { + multipartRequest = request + } + else if(request.respondsTo('getRequest') && request.getRequest() instanceof MultipartHttpServletRequest) { + multipartRequest = request.getRequest() + } + else { + throw new IllegalStateException("Request is a ${request.getClass()} and does not wrap a MultipartHttpServletRequest") + } + //assert request instanceof MultipartHttpServletRequest + def file = multipartRequest.getFile('fileUpload') as CommonsMultipartFile log.debug("File: ${file.originalFilename}, Type: ${file.contentType}") def files = [] as List<File>; - def directories = [] as List<File>; - def messages = "" as String + def messages = [] as List<String>; Sandbox sandbox = sandboxManager.getSandbox(wiki.path) // no SVN - if(! sandbox) { + if(!sandbox) { log.error("No sandbox found for wiki path=${wiki.path}") - messages += "SVN not accessible.\n" - + messages << "SVN not accessible." + } // file empty - } else if(file.empty) { - messages += "The file ${file.originalFilename} is empty or the file " - messages += "path or name is incorrect.\n" - + else if(file.empty) { + messages << "The file ${file.originalFilename} is empty or the file " + + "path or name is incorrect." + } // file too big - } else if (file.size > (50*1024*1024)) { - messages += "The file ${file.originalFilename} is bigger than 50MB.\n" - + else if (file.size > (50*1024*1024)) { + messages << "The file ${file.originalFilename} is bigger than 50MB." + } // ZIP file - } else if (params.unzip == "on") { - // open the zip file stream - InputStream theFile = file.getInputStream(); - ZipInputStream stream = new ZipInputStream(theFile); - messages += "Processing ZIP file:\n" - - try { - - // now iterate through each item in the stream. The get next - // entry call will return a ZipEntry for each file in the - // stream - byte[] buffer = new byte[2048]; - ZipEntry entry; - while((entry = stream.getNextEntry()) != null) { - String s = String.format("Read ZIP Entry: %s, length %d, date %TD", - entry.getName(), entry.getSize(), - new Date(entry.getTime())); - log.debug(System.out.println(s)); - messages += s + "\n" - - File pageFile = new File(thisPage.pageFileDir, entry.getName()); - if (pageFile.exists() && params.overwrite != "on") { - messages += "File: ${pageFile} already exists.\n" - continue; - } - - File pageFileNewDir = pageFile.getParentFile(); - if (!pageFileNewDir.exists()) { - // create missing directories - pageFileNewDir.mkdirs(); - messages += "Created directory: ${pageFileNewDir}.\n" - directories.add pageFileNewDir - } - // Once we get the entry from the stream, the stream is - // positioned read to read the raw data, and we keep - // reading until read returns 0 or less. - FileOutputStream output = null; - try { - output = new FileOutputStream(pageFile); - int len = 0; - while ((len = stream.read(buffer)) > 0) { - output.write(buffer, 0, len); - } - messages += "Written file: ${pageFile}.\n" - files.add pageFile - - } catch (Exception e) { - log.debug(e) - messages += e.getMessage() + "\n" - files.remove pageFile - - } finally { - // we must always close the output file - if(output!=null) output.close() - } - } - - } catch (Exception e) { - log.debug(e) - messages += e.getMessage() + "\n" - directories.clear() - files.clear() - - } finally { - // we must always close the zip file. - stream.close() + else if (params.unpack == "on") { + file.getInputStream().withStream { stream -> + pageService.unpackArchive(stream, file.originalFilename, + thisPage.pageFileDir, files, messages, + (params.overwrite == 'on')) } - + } // non Zip file - } else { + else { File newFile = new File(thisPage.pageFileDir, file.originalFilename) if (newFile.exists() && params.overwrite != "on") { - messages += "File: ${newFile} already exists.\n" + messages << "File: ${newFile} already exists." } else { file.transferTo(newFile) - messages += "Written file: ${newFile}\n" + messages << "Written file: ${newFile}" files.add newFile } } - messages += "\n" - // add directories to SVN - directories.each { - try { - def params2 = [id: thisPage.id, path: it] - def thisPage2 = pageService.analyse(params2) ?: [ : ] - sandbox.addEntry(it) - Set<String> dirtyList = dependenciesService.created(wiki, it) - yamService.queueRegenerations(dirtyList, thisPage2) - messages += "Added directory to SVN: ${it}.\n" - } catch (Exception e) { - log.debug(e) - messages += e.getMessage() + "\n" - } + // sort the list of files. As the path to a file's parent dir is always a + // prefix of the path to the file itself, this nicely ensures that we will + // add directories before trying to add the files they contain. + Collections.sort(files.unique()) - } + log.debug("Files/directories to add: ${files}") // collecting urls for indexing def urlsToIndex = [] as List<String>; - // add files to SVN + // non-recursively add each file (or dir) to svn files.each { - try { - def params2 = [id: thisPage.id, path: it] - def thisPage2 = pageService.analyse(params2) ?: [ : ] - sandbox.addEntry(it) - Set<String> dirtyList = dependenciesService.created(wiki, it) - yamService.queueRegenerations(dirtyList, thisPage2) - messages += "Added file to SVN: ${it}.\n" - - // add it to the urlsToIndex - urlsToIndex.add it.toURL().toString() - - } catch (Exception e) { - log.debug(e) - messages += e.getMessage() + "\n" + // don't try and add things that are already added + if(sandbox.doStatus(it, false).contentsStatus?.is( + SVNStatusType.STATUS_UNVERSIONED)) { + try { + def params2 = [id: thisPage.id, path: it] + def thisPage2 = pageService.analyse(params2) ?: [ : ] + sandbox.addEntry(it, false) // non-recursive add + Set<String> dirtyList = dependenciesService.created(wiki, it) + yamService.queueRegenerations(dirtyList, thisPage2) + messages << "Added ${it.isDirectory() ? 'directory' : 'file'} to SVN: ${it}." + if(!it.isDirectory()) { + // add it to the urlsToIndex + urlsToIndex.add it.toURL().toString() + } + } + catch (Exception e) { + log.debug(e) + messages << e.getMessage() + } } } + // in server mode, do a commit if(conf.mode == "server") { - def itemsToCommit = directories + files - sandbox.commit(itemsToCommit as File[], false, "CoW upload") + sandbox.commit(files as File[], false, "CoW upload") } // indexing should run only if user has set it to true @@ -344,16 +288,18 @@ indexingService.index(request, urlsToIndex, params.id, true) } - // shorten paths in the messages - messages = messages.replaceAll("\\Q" + thisPage.pageFileDir + "\\E", - (thisPage.pagePathDir - File.separator)) - // encode as HTML to avoid malicious javascript? - messages = messages.encodeAsHTML() - messages = messages.replaceAll("\n", "\n<br>") + def message = messages.collect { + // shorten paths in the messages + '<li>' + it.replaceAll("\\Q" + thisPage.pageFileDir + "\\E", + (thisPage.pagePathDir - File.separator)).encodeAsHTML() + '</li>\n' + }.join('\n') + if(message) { + message = '<ul>' + message + '</ul>' + } if(! files.empty) { - flash.message = "Upload successful.<br><br>" + messages; + flash.message = "Upload successful.<br><br>" + message; } else { - flash.message = "Upload failed.<br><br>" + messages; + flash.message = "Upload failed.<br><br>" + message; } } // upload (GWT) Modified: trunk/cow/grails-app/services/PageService.groovy =================================================================== --- trunk/cow/grails-app/services/PageService.groovy 2009-11-03 16:04:13 UTC (rev 1786) +++ trunk/cow/grails-app/services/PageService.groovy 2009-11-04 16:42:41 UTC (rev 1787) @@ -22,6 +22,10 @@ import gate.yam.convert.HtmlToYamConverter import gate.versioning.svnkit.SandboxManager import gate.versioning.svnkit.Sandbox +import org.apache.tools.bzip2.CBZip2InputStream +import org.apache.tools.tar.TarInputStream +import java.util.zip.GZIPInputStream +import java.util.zip.ZipInputStream /** Tasks related to page requests. */ class PageService { @@ -752,6 +756,103 @@ return dirs } // getDirList(File, Wiki) + /** + * Unpack the given archive file into the given directory. + * + * @param stream the input stream from which to read the archive + * @param archiveFileName the original file name of the archive, used to + * determine the archive type (.zip, .tar.gz, .tar.bz2) + * @param baseDir the directory into which the files are to be unpacked + * @param addedFiles a list to which the File objects corresponding to added + * files and directories will be added + * @param messages a list to which messages generated during processing will + * be added + * @param overwrite should we overwrite existing files? + */ + void unpackArchive(InputStream stream, String archiveFileName, File baseDir, + List addedFiles, List messages, boolean overwrite) + throws IOException { + def archiveStream + switch(archiveFileName) { + case ~/(?i).*\.zip$/: + archiveStream = new ZipInputStream(stream) + break + + case ~/(?i).*\.tar$/: + archiveStream = new TarInputStream(stream) + break + + case ~/(?i).*\.t(?:ar\.)?gz$/: + archiveStream = new TarInputStream(new GZIPInputStream(stream)) + break + + case ~/(?i).*\.t(?:ar\.)?bz2$/: + // consume the first two bytes and check they are 'BZ' + if(stream.read() != 66 || stream.read() != 90) { + throw new IOException("Invalid bzip2 archive") + } + archiveStream = new TarInputStream(new CBZip2InputStream(stream)) + break + + default: + throw new IOException("Unrecognised archive type") + } + + // I like groovy loose typing - TarInputStream/TarEntry and + // ZipInputStream/ZipEntry look like the same kind of duck... + messages << "Processing archive file:" + def numFiles = 0 + + try { + def entry + while((entry = archiveStream.getNextEntry()) != null) { + def name = entry.name + // paranoid check - strip any leading / from entry names + if(name.startsWith('/')) { + name -= '/' + if(!name) { + continue + } + } + def targetFile = new File(baseDir, name) + if(entry.isDirectory()) { + if(targetFile.exists() && !targetFile.isDirectory()) { + messages << "Cannot create directory ${name} as a file of the same name already exists" + continue + } + targetFile.mkdirs() + } + else { + if(!overwrite && targetFile.exists()) { + messages << "File ${name} already exists." + continue + } + File dir = targetFile.getParentFile() + if(dir.exists() && !dir.isDirectory()) { + messages << "Cannot create parent directory for ${name} as a file of the same name already exists" + continue + } + def parent = targetFile.getParentFile() + parent.mkdirs() + addedFiles << parent + + try { + targetFile.withOutputStream() { it << archiveStream } + numFiles += 1 + } + catch(IOException e) { + messages << "Error processing archive entry ${name}: ${e.message}" + continue + } + } + addedFiles << targetFile + } + } + finally { + archiveStream.close() + } + } + /** Config */ static Map conf = CowUtils.config.gate.cow Modified: trunk/cow/selenium/tests/upload-server.html =================================================================== --- trunk/cow/selenium/tests/upload-server.html 2009-11-03 16:04:13 UTC (rev 1786) +++ trunk/cow/selenium/tests/upload-server.html 2009-11-04 16:42:41 UTC (rev 1787) @@ -160,7 +160,7 @@ </tr> <tr> <td>click</td> - <td>unzip</td> + <td>unpack</td> <td></td> </tr> <tr> Modified: trunk/cow/selenium/tests/upload.html =================================================================== --- trunk/cow/selenium/tests/upload.html 2009-11-03 16:04:13 UTC (rev 1786) +++ trunk/cow/selenium/tests/upload.html 2009-11-04 16:42:41 UTC (rev 1787) @@ -128,7 +128,7 @@ </tr> <tr> <td>click</td> - <td>unzip</td> + <td>unpack</td> <td></td> </tr> <tr> Modified: trunk/cow/src/java/gate/cow/gwt/client/Upload.java =================================================================== --- trunk/cow/src/java/gate/cow/gwt/client/Upload.java 2009-11-03 16:04:13 UTC (rev 1786) +++ trunk/cow/src/java/gate/cow/gwt/client/Upload.java 2009-11-04 16:42:41 UTC (rev 1787) @@ -88,13 +88,14 @@ final FileUpload fileUpload = new FileUpload(); fileUpload.setName("fileUpload"); fileUpload.setTitle( - "Choose a file or ZIP archive to upload in the current directory."); + "Choose a file or archive to upload in the current directory. " + + "Supported archive formats are zip, tar, tar.gz or tar.bz2"); hPanel.add(fileUpload); - CheckBox unzipCheckBox = new CheckBox("Unzip"); - unzipCheckBox.setName("unzip"); + CheckBox unzipCheckBox = new CheckBox("Unpack"); + unzipCheckBox.setName("unpack"); unzipCheckBox.setChecked(false); unzipCheckBox.ensureDebugId("unzipCheckBox"); - unzipCheckBox.setTitle("Unzip the file in the current directory."); + unzipCheckBox.setTitle("Unpack the archive file in the current directory."); hPanel.add(unzipCheckBox); CheckBox overwriteCheckBox = new CheckBox("Overwrite"); overwriteCheckBox.setName("overwrite"); Modified: trunk/cow/src/java/gate/versioning/svnkit/Sandbox.java =================================================================== --- trunk/cow/src/java/gate/versioning/svnkit/Sandbox.java 2009-11-03 16:04:13 UTC (rev 1786) +++ trunk/cow/src/java/gate/versioning/svnkit/Sandbox.java 2009-11-04 16:42:41 UTC (rev 1787) @@ -521,11 +521,13 @@ * @param recurse if true, and wcPath names a directory, recursively add all * the directories contents. If false, just add the directory itself * but not its contents. Has no effect if wcPath names a normal - * file. + * file. Note this is different from the recurse parameter to most + * other methods in this class, where non-recursive includes files in + * the directory but not subdirectories. */ synchronized public void addEntry(File wcPath, boolean recurse) throws SVNException { clientManager.getWCClient().doAdd(wcPath, false, false, false, - SVNDepth.fromRecurse(recurse), false, false); + (recurse ? SVNDepth.INFINITY : SVNDepth.EMPTY), false, false); } // addEntry /** This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. ------------------------------------------------------------------------------ Let Crystal Reports handle the reporting - Free Crystal Reports 2008 30-Day trial. Simplify your report design, integration and deployment - and focus on what you do best, core application coding. Discover what's new with Crystal Reports now. http://p.sf.net/sfu/bobj-july _______________________________________________ gatewiki-commits mailing list gatewiki-commits@... https://lists.sourceforge.net/lists/listinfo/gatewiki-commits |
| Free embeddable forum powered by Nabble | Forum Help |
