diff --git a/README.md b/README.md index 934db39edb..d6f1195321 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,7 @@ There are several branches in the repository. Here are the ones that you should - [Gradle 8.10] (https://gradle.org/install/) Gradle is an open source build automation system. - - [Tomcat 9] (https://tomcat.apache.org/download-90.cgi) + - [Tomcat 11] (https://tomcat.apache.org/download-11.cgi) Apache Tomcat is an open source software implementation of the Java Servlet and JavaServer Pages technologies. - [Node v18] (https://nodejs.org/) diff --git a/build.gradle b/build.gradle index a5c6b07ac2..ec168ddf15 100644 --- a/build.gradle +++ b/build.gradle @@ -1,24 +1,8 @@ ext.fireflyPath = rootDir.path - -allprojects { - defaultTasks 'preselect' -} +apply from: "$fireflyPath/buildScript/base.gincl" subprojects { apply plugin: "java" apply from: "$fireflyPath/buildScript/global.gincl" apply from: "$fireflyPath/buildScript/tasks.gincl" } - -task purge { - description= 'Remove all build files from this project, including node_module.' - - doLast { - println('Removes all firefly build directories.') - delete "${fireflyPath}/build", "${fireflyPath}/jars/build", "${fireflyPath}/node_modules" - } -} - -task preselect { - description= 'A placeholder task to designate the task to run for each project. Similar to defaultTasks.' -} \ No newline at end of file diff --git a/buildScript/base.gincl b/buildScript/base.gincl new file mode 100644 index 0000000000..049960e07c --- /dev/null +++ b/buildScript/base.gincl @@ -0,0 +1,48 @@ +import java.nio.file.Path +import java.nio.file.Paths +import java.nio.file.Files + +def deleteLink(String filePath) { + Path path = Paths.get(filePath) + if (Files.exists(path) && Files.isSymbolicLink(path)) { + println("Deleting symlink: $filePath") + Files.delete(path) // deletes symlink only + } +} + +allprojects { + defaultTasks 'preselect' +} + +task preselect { + description= 'A placeholder task to designate the task to run for each project. Similar to defaultTasks.' +} + +task purge { + description = 'Removes all build files from this project, including node_modules, Maven cache, and Gradle cache.' + + doLast { + println('Removing build directories, including node_modules, Maven cache, and Gradle cache.') + delete "${rootDir}/build", + "${rootDir}/jars/build", + "${rootDir}/node_modules", + "${rootDir}/.gradle" + + println('Removing Maven and Gradle caches...') + delete "${System.getProperty('user.home')}/.m2/repository", + "${System.getProperty('user.home')}/.gradle/caches" + + deleteLink("${rootDir}/yarn.lock") + deleteLink("${rootDir}/package.json") + deleteLink("${rootDir}/config/test") + + } +} + +task clean { + description= 'Removes all build files except installed libraries, e.g. node_modules, Maven cache.' + doLast { + println('Removing all build directories.') + delete "${rootDir}/build", "${rootDir}/jars/build" + } +} diff --git a/buildScript/dependencies.gradle b/buildScript/dependencies.gradle index 87b223e486..aa53062d68 100644 --- a/buildScript/dependencies.gradle +++ b/buildScript/dependencies.gradle @@ -7,15 +7,18 @@ def libPath = "$fireflyPath/jars" //project.ext.properties.each{ k, v -> println "${k}:${v}\n" } repositories { - mavenCentral() maven { url "https://maven.repository.redhat.com/ga/" + content { + includeGroup "org.josso" + } } flatDir { name 'firefly_repo' dirs "$libPath/build", "$libPath/starlink" } + mavenCentral() } configurations { @@ -35,7 +38,7 @@ dependencies { // ============== from maven central =============== // apache commons - implementation 'commons-fileupload:commons-fileupload:1.2.2', 'commons-io:commons-io:2.4', 'commons-lang:commons-lang:2.2', 'org.apache.commons:commons-csv:1.0', 'org.apache.commons:commons-compress:1.28.0' + implementation 'commons-io:commons-io:2.4', 'commons-lang:commons-lang:2.2', 'org.apache.commons:commons-csv:1.0', 'org.apache.commons:commons-compress:1.28.0' // axis Web Services SOAP implementation 'axis:axis:1.4' @@ -78,7 +81,10 @@ dependencies { implementation 'org.reflections:reflections:0.9.11' // servlet API - implementation 'javax.servlet:javax.servlet-api:3.1.0' + implementation 'jakarta.servlet:jakarta.servlet-api:6.1.0' + + // FileUpload Jakarta fork for servlet api 6.0+ + implementation 'org.apache.commons:commons-fileupload2-jakarta:2.0.0-M1' // spring jdbc implementation 'org.springframework:spring-jdbc:2.5.5' @@ -91,7 +97,7 @@ dependencies { implementation 'javax.validation:validation-api:1.1.0.Final' // websocket - implementation 'javax.websocket:javax.websocket-api:1.1' + implementation 'jakarta.platform:jakarta.jakartaee-web-api:10.0.0' // junit for java test implementation 'junit:junit:4.12' diff --git a/config/web.xml b/config/web.xml index c4af0fd042..9f32eadfee 100644 --- a/config/web.xml +++ b/config/web.xml @@ -27,23 +27,6 @@ /firefly_loader.js - - - GZipFilter - net.sf.ehcache.constructs.web.filter.GzipFilter - - - GZipFilter - *.html - *.css - *.js - /sticky/CmdSrv - /CmdSrv/sync - /CmdSrv/async - /CmdSrv/async/* - /sticky/firefly/events - - CorsFilter diff --git a/docker/Dockerfile b/docker/Dockerfile index ca65b0f41c..cd77e161bd 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -38,7 +38,7 @@ RUN apt-get update \ # cleanup && rm -rf /var/lib/apt/lists/*; -COPY --from=tomcat:9.0-jre21-temurin-jammy /usr/local/tomcat /usr/local/tomcat +COPY --from=tomcat:11.0.11-jre21-temurin-jammy /usr/local/tomcat /usr/local/tomcat ENV CATALINA_HOME=/usr/local/tomcat \ @@ -106,7 +106,7 @@ RUN gradle -Penv=${env} -PBranchOverride=${BranchOverride} ${target} # /external : default external data directory visible to Firefly -FROM tomcat:9.0-jre21-temurin-jammy +FROM tomcat:11.0.11-jre21-temurin-jammy ARG build_dir ARG user=tomcat @@ -164,6 +164,13 @@ COPY firefly/docker/local.xml conf/Catalina/localhost #copy all wars, typically there should only be one COPY --from=builder /opt/work/${build_dir}/build/dist/*.war ${CATALINA_HOME}/webapps-ref/ +# Turn on GZIP compression by adding these attributes to the Tomcat connector: +# - compression="on" → turn on response compression +# - useSendfile="false" → avoids bypassing compression for static files +# - compressibleMimeType → list of text-based formats to compress (HTML, CSS, JS, JSON, VOTable, XML, CSV, SVG, Fonts) +RUN sed -i 's/> posted file * * @author loi @@ -80,8 +82,6 @@ public class AnyFileUpload extends BaseHttpServlet { FILE_NAME, CACHE_KEY, WORKSPACE_PUT, WS_CMD, ANALYZER_ID,HIPS_CACHE, WEB_PLOT_REQUEST, FILE_ANALYSIS, ServerParams.COMMAND); - - protected void processRequest(HttpServletRequest req, HttpServletResponse res) throws Exception { doFileUpload(req, res); } @@ -90,31 +90,35 @@ public static void doFileUpload(HttpServletRequest req, HttpServletResponse res) StopWatch.getInstance().start("doFileUpload"); - FileItemStream uploadedItem = null; + FileItemInput uploadedItem = null; // processes the parameters... SrvParam sp = new SrvParam(req.getParameterMap()); - if (ServletFileUpload.isMultipartContent(req)) { // this is a multipart request... extract params from parts - ServletFileUpload upload = new ServletFileUpload( new DiskFileItemFactory(64 * 1024, getUploadDir("tmpDiskStore"))); // set factory to raise in-memory usage - for (FileItemIterator iter = upload.getItemIterator(req); iter.hasNext(); ) { - FileItemStream item = iter.next(); + if (JakartaServletFileUpload.isMultipartContent(req)) { // this is a multipart request... extract params from parts + + DiskFileItemFactory factory = DiskFileItemFactory.builder() + .setBufferSize(1024 * 1024).get(); // 1MB threshold before writing to disk + + JakartaServletDiskFileUpload upload = new JakartaServletDiskFileUpload(factory); + + FileItemInputIterator iter = upload.getItemIterator(req); + while (iter.hasNext()) { + FileItemInput item = iter.next(); if (item.isFormField()) { - sp.setParam(item.getFieldName(), FileUtil.readFile(item.openStream())); + sp.setParam(item.getFieldName(), FileUtil.readFile(item.getInputStream())); } else { uploadedItem = item; - break; - // file should be the last param. param after file will be ignored. + break; // file should be the last param } - } - } + } } // handle upload file request... results in saved as an UploadFileInfo String analysisType = sp.getOptional(FILE_ANALYSIS); boolean analyzeFile= analysisType != null && !analysisType.equalsIgnoreCase("false"); try { - Result result= retrieveFile(sp,uploadedItem); + Result result= retrieveFile(sp, uploadedItem); UploadFileInfo uploadFileInfo = result.uploadFileInfo; FileInfo statusFileInfo= result.statusFileInfo; int responseCode= statusFileInfo.getResponseCode(); @@ -174,10 +178,10 @@ private static boolean isUrlFail(FileInfo statusFileInfo) { return (file!=null && responseCode!=200 && responseCode!=304); } - private static UploadFileInfo makeUploadFileInfo(FileInfo statusFileInfo, String fname) { File file= statusFileInfo.getFile(); - return new UploadFileInfo(ServerContext.replaceWithPrefix(file), file, fname!=null ? fname : file.getName(), + return new UploadFileInfo(ServerContext.replaceWithPrefix(file), file, + fname!=null ? fname : file.getName(), statusFileInfo.getContentType(), statusFileInfo.getResponseCode()); } @@ -193,7 +197,6 @@ private static FileAnalysisReport.ReportType getReportType(String type) { } catch (Exception e) { return FileAnalysisReport.ReportType.Details; } - } private static UploadFileInfo getFileFromWorkspace(SrvParam sp) throws IOException, FailedRequestException { @@ -204,7 +207,6 @@ private static UploadFileInfo getFileFromWorkspace(SrvParam sp) throws IOExcepti return new UploadFileInfo(rPathInfo, uf, fileName, null); } - private static String callAnalysis(SrvParam sp, FileInfo statusFileInfo, UploadFileInfo uploadFileInfo, String fileCacheKey) throws Exception { @@ -222,9 +224,8 @@ private static String callAnalysis(SrvParam sp, FileInfo statusFileInfo, return returnVal; } + private static Result retrieveFile(SrvParam sp, FileItemInput uploadedItem) throws Exception { - private static Result retrieveFile(SrvParam sp, FileItemStream uploadedItem) throws Exception { - String wsCmd = sp.getOptional(WS_CMD); String fromUrl = sp.getOptional(URL); WebPlotRequest fromWPR= sp.getOptionalWebPlotRequest(WEB_PLOT_REQUEST); @@ -271,11 +272,10 @@ private static Result retrieveFile(SrvParam sp, FileItemStream uploadedItem) thr // it's a stream from multipart... write it to disk String name = uploadedItem.getName(); File tmpFile = File.createTempFile("upload_", "_" + name, getSessUploadDir(sp.convertToServerRequest())); - FileUtil.writeToFile(uploadedItem.openStream(), tmpFile, (current) -> updateFeedback(name, current)); + FileUtil.writeToFile(uploadedItem.getInputStream(), tmpFile, (current) -> updateFeedback(name, current)); uploadFileInfo = new UploadFileInfo(ServerContext.replaceWithPrefix(tmpFile), tmpFile, name, uploadedItem.getContentType()); statusFileInfo= new FileInfo(uploadFileInfo.getFile()); - } - else { + } else { throw new IllegalArgumentException("Invalid parameters to AnyFileUpload"); } return new Result(statusFileInfo,uploadFileInfo); diff --git a/src/firefly/java/edu/caltech/ipac/firefly/server/servlets/BaseHttpServlet.java b/src/firefly/java/edu/caltech/ipac/firefly/server/servlets/BaseHttpServlet.java index 4cc449f856..89af4b9e1a 100644 --- a/src/firefly/java/edu/caltech/ipac/firefly/server/servlets/BaseHttpServlet.java +++ b/src/firefly/java/edu/caltech/ipac/firefly/server/servlets/BaseHttpServlet.java @@ -7,12 +7,12 @@ import edu.caltech.ipac.firefly.ui.creator.CommonParams; import edu.caltech.ipac.util.ComparisonUtil; -import javax.servlet.ServletConfig; -import javax.servlet.ServletException; -import javax.servlet.http.Cookie; -import javax.servlet.http.HttpServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; +import jakarta.servlet.ServletConfig; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.Cookie; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; import java.io.IOException; import static edu.caltech.ipac.firefly.server.filters.CorsFilter.enableCors; diff --git a/src/firefly/java/edu/caltech/ipac/firefly/server/servlets/CommandService.java b/src/firefly/java/edu/caltech/ipac/firefly/server/servlets/CommandService.java index 64329e1d1d..c62005706c 100644 --- a/src/firefly/java/edu/caltech/ipac/firefly/server/servlets/CommandService.java +++ b/src/firefly/java/edu/caltech/ipac/firefly/server/servlets/CommandService.java @@ -10,8 +10,8 @@ import edu.caltech.ipac.firefly.server.util.Logger; import org.json.simple.JSONObject; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; diff --git a/src/firefly/java/edu/caltech/ipac/firefly/server/servlets/HealthCheck.java b/src/firefly/java/edu/caltech/ipac/firefly/server/servlets/HealthCheck.java index 6e7f30ebc0..5796040b74 100644 --- a/src/firefly/java/edu/caltech/ipac/firefly/server/servlets/HealthCheck.java +++ b/src/firefly/java/edu/caltech/ipac/firefly/server/servlets/HealthCheck.java @@ -8,8 +8,8 @@ import edu.caltech.ipac.util.FileUtil; import edu.caltech.ipac.util.StringUtils; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; /** * Check application health by performing a simple task, then return HTTP code 200 if succeed. diff --git a/src/firefly/java/edu/caltech/ipac/firefly/server/servlets/HttpServCommands.java b/src/firefly/java/edu/caltech/ipac/firefly/server/servlets/HttpServCommands.java index 1aebfd3004..2c08ddc41e 100644 --- a/src/firefly/java/edu/caltech/ipac/firefly/server/servlets/HttpServCommands.java +++ b/src/firefly/java/edu/caltech/ipac/firefly/server/servlets/HttpServCommands.java @@ -18,8 +18,8 @@ import edu.caltech.ipac.table.TableUtil; import org.json.simple.JSONObject; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; import java.io.File; import java.io.FileOutputStream; diff --git a/src/firefly/java/edu/caltech/ipac/firefly/server/servlets/MultiPartHandler.java b/src/firefly/java/edu/caltech/ipac/firefly/server/servlets/MultiPartHandler.java deleted file mode 100644 index 9b6be4f5c3..0000000000 --- a/src/firefly/java/edu/caltech/ipac/firefly/server/servlets/MultiPartHandler.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * License information at https://github.com/Caltech-IPAC/firefly/blob/master/License.txt - */ -package edu.caltech.ipac.firefly.server.servlets; - -import edu.caltech.ipac.firefly.server.util.Logger; -import edu.caltech.ipac.firefly.server.util.multipart.MultiPartData; -import edu.caltech.ipac.util.StringUtils; -import edu.caltech.ipac.util.cache.CacheManager; - -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - - -/** - * This servlet take a multipart request and extract the files and parameters from it. - * The data are stored in the UserCache as a MultiPartData. - * The key is returned to the caller(HttpResponse). - * The files are stored in a temporary work directory backed by - * the TYPE_TEMP_FILE cache... which remove the older file(s) if it exceeds a - * pre-configured number. So, you do not need to handle the deletion of the - * uploaded files. But, there's a chance that the files are no longer there, - * after a long period of time. - * - * Date: July 27, 2010 - * - * @author loi - * @version $Id: MultiPartHandler.java,v 1.3 2012/03/23 19:12:37 roby Exp $ - */ -public class MultiPartHandler extends BaseHttpServlet { - private static final Logger.LoggerImpl LOG = Logger.getLogger(); - - - protected void processRequest(HttpServletRequest req, HttpServletResponse res) throws Exception { - - try { - MultiPartData data= MultipartDataUtil.handleRequest(req); - CacheManager.getUserCache().put(data.getCacheKey(), data); - LOG.info("Multipart request processed.", - "File(s) uploaded: " + StringUtils.toString(data.getFiles()), - "Form parameters : " + data.getParams()); - sendReturnMsg(res, 200, "uploaded successfully", data.getCacheKey().toString()); - } catch (Exception e) { - sendReturnMsg(res, 500, e.getMessage(), null); - } - -// test code.. -// if (doTest) { -// MultiPartPostBuilder builder = new MultiPartPostBuilder( -// "http://localhost:8080/applications/Spitzer/SHA/servlet/Firefly_MultiPartHandler"); -// builder.addParam("dummy1", "boo1"); -// builder.addParam("dummy2", "boo2"); -// builder.addParam("dummy3", "boo3"); -// for(UploadFileInfo fi : data.getFiles()) { -// builder.addFile(fi.getPname(), fi.getFile()); -// } -// StringWriter sw = new StringWriter(); -// MultiPartPostBuilder.MultiPartRespnse pres = builder.post(sw); -// LOG.briefDebug("uploaded status: " + pres.getStatusMsg()); -// LOG.debug("uploaded response: " + sw.toString()); -// } - } -} diff --git a/src/firefly/java/edu/caltech/ipac/firefly/server/servlets/MultipartDataUtil.java b/src/firefly/java/edu/caltech/ipac/firefly/server/servlets/MultipartDataUtil.java deleted file mode 100644 index 57528d0eab..0000000000 --- a/src/firefly/java/edu/caltech/ipac/firefly/server/servlets/MultipartDataUtil.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * License information at https://github.com/Caltech-IPAC/firefly/blob/master/License.txt - */ -package edu.caltech.ipac.firefly.server.servlets; -/** - * User: roby - * Date: 3/16/12 - * Time: 1:45 PM - */ - - -import edu.caltech.ipac.firefly.server.util.QueryUtil; -import edu.caltech.ipac.firefly.server.util.multipart.MultiPartData; -import edu.caltech.ipac.util.cache.CacheManager; -import edu.caltech.ipac.util.cache.StringKey; -import org.apache.commons.fileupload.FileItem; -import org.apache.commons.fileupload.disk.DiskFileItemFactory; -import org.apache.commons.fileupload.servlet.ServletFileUpload; - -import javax.servlet.http.HttpServletRequest; -import java.io.File; -import java.util.Iterator; -import java.util.List; - -/** - * @author Trey Roby - */ -public class MultipartDataUtil { - - public static MultiPartData handleRequest(HttpServletRequest req) throws Exception { - return handleRequest(new StringKey("MultiPartHandler", System.currentTimeMillis()), req); - } - - public static MultiPartData handleRequest(StringKey key, HttpServletRequest req) throws Exception { - - // Create a factory for disk-based file items - DiskFileItemFactory factory = new DiskFileItemFactory(); - - // Create a new file upload handler - ServletFileUpload upload = new ServletFileUpload(factory); - - // Parse the request - List /* FileItem */ items = upload.parseRequest(req); - - MultiPartData data = new MultiPartData(key); - - - // Process the uploaded items - Iterator iter = items.iterator(); - while (iter.hasNext()) { - FileItem item = (FileItem) iter.next(); - - if (item.isFormField()) { - String name = item.getFieldName(); - String value = item.getString(); - data.addParam(name, value); - } else { - String fieldName = item.getFieldName(); - String fileName = item.getName(); - String contentType = item.getContentType(); - File uf = new File(QueryUtil.getTempDir(), System.currentTimeMillis() + ".upload"); - item.write(uf); - data.addFile(fieldName, uf, fileName, contentType); - StringKey fileKey= new StringKey(fileName, System.currentTimeMillis()); - CacheManager.getLocal().put(fileKey, uf); - } - } - return data; - } - - - - - -} - diff --git a/src/firefly/java/edu/caltech/ipac/firefly/server/servlets/ServerStatus.java b/src/firefly/java/edu/caltech/ipac/firefly/server/servlets/ServerStatus.java index 0a25530190..28ac4cfc42 100644 --- a/src/firefly/java/edu/caltech/ipac/firefly/server/servlets/ServerStatus.java +++ b/src/firefly/java/edu/caltech/ipac/firefly/server/servlets/ServerStatus.java @@ -24,8 +24,8 @@ import net.sf.ehcache.distribution.CachePeer; import net.sf.ehcache.statistics.StatisticsGateway; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; import java.io.File; import java.io.PrintWriter; import java.rmi.RemoteException; diff --git a/src/firefly/java/edu/caltech/ipac/firefly/server/util/VersionUtil.java b/src/firefly/java/edu/caltech/ipac/firefly/server/util/VersionUtil.java index f1c1f16f7b..ab6455d50c 100644 --- a/src/firefly/java/edu/caltech/ipac/firefly/server/util/VersionUtil.java +++ b/src/firefly/java/edu/caltech/ipac/firefly/server/util/VersionUtil.java @@ -8,7 +8,7 @@ import edu.caltech.ipac.util.KeyVal; import edu.caltech.ipac.util.StringUtils; -import javax.servlet.ServletContext; +import jakarta.servlet.ServletContext; import java.io.File; import java.io.FileInputStream; import java.io.IOException;