use temp file for large POSTs

core/java/src/net/i2p/util/EepGet.java

@@ -3,6 +3,7 @@ package net.i2p.util;
 import java.io.BufferedInputStream;
 import java.io.BufferedReader;
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
@@ -59,6 +60,7 @@ public class EepGet {
     protected String _actualURL;
     private String _postData;
     private byte[] _postBinaryData;
+    private File _postDataFile;
     private boolean _allowCaching;
     protected final List<StatusListener> _listeners;
     protected List<String> _extraHeaders;
@@ -1440,8 +1442,16 @@ public class EepGet {
             timeout.setSocket(_proxy);
         
         _proxyOut.write(DataHelper.getUTF8(req));
-        if (_postBinaryData != null)
+        if (_postBinaryData != null) {
             _proxyOut.write(_postBinaryData);
+        } else if (_postDataFile != null) {
+            InputStream in = new FileInputStream(_postDataFile);
+            try {
+                DataHelper.copy(in, _proxyOut);
+            } finally {
+                try { in.close(); } catch (IOException ioe) {}
+            }
+        }
         _proxyOut.flush();
         
         if (_log.shouldLog(Log.DEBUG))
@@ -1452,7 +1462,8 @@ public class EepGet {
         StringBuilder buf = new StringBuilder(2048);
         boolean post = false;
         if ((_postData != null && _postData.length() > 0) ||
-            (_postBinaryData != null && _postBinaryData.length > 0))
+            (_postBinaryData != null && _postBinaryData.length > 0) ||
+            (_postDataFile != null && _postDataFile.length() > 0))
             post = true;
         URI url;
         try {
@@ -1533,8 +1544,10 @@ public class EepGet {
             buf.append("Content-length: ");
             if (_postData != null)
                 buf.append(_postData.length());
-            else
+            else if (_postBinaryData != null)
                 buf.append(_postBinaryData.length);
+            else
+                buf.append(_postDataFile.length());
             buf.append("\r\n");
         }
         // This will be replaced if we are going through I2PTunnelHTTPClient
@@ -1687,7 +1700,7 @@ public class EepGet {
      *  @since 0.9.67
      */
     protected void setPostData(String contentType, String data) {
-        if (_postData != null || _postBinaryData != null)
+        if (_postData != null || _postBinaryData != null || _postDataFile != null)
             throw new IllegalStateException();
         addHeader("Content-Type", contentType);
         _postData = data;
@@ -1701,12 +1714,26 @@ public class EepGet {
      *  @since 0.9.67
      */
     protected void setPostData(String contentType, byte[] data) {
-        if (_postData != null || _postBinaryData != null)
+        if (_postData != null || _postBinaryData != null || _postDataFile != null)
             throw new IllegalStateException();
         addHeader("Content-Type", contentType);
         _postBinaryData = data;
     }
 
+    /**
+     *  Set post data.
+     *  Must be called before fetch().
+     *
+     *  @throws IllegalStateException if already set
+     *  @since 0.9.67
+     */
+    protected void setPostData(String contentType, File data) {
+        if (_postData != null || _postBinaryData != null || _postDataFile != null)
+            throw new IllegalStateException();
+        addHeader("Content-Type", contentType);
+        _postDataFile = data;
+    }
+
     /**
      *  Parse the args in an authentication header.
      *

core/java/src/net/i2p/util/EepPost.java

@@ -5,6 +5,7 @@ import java.io.BufferedReader;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileInputStream;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
@@ -24,6 +25,7 @@ import net.i2p.data.DataHelper;
 /**
  * Extends EepGet for POST.
  * Adapted from old jrandom EepPost, removed 2012 as unused.
+ * Ref: RFC 7578
  *
  * @since 0.9.67
  */
@@ -45,7 +47,8 @@ public class EepPost extends EepGet {
      * value is posted for that particular field.  Multiple values for one 
      * field name is not currently supported.
      *
-     * Warning: Files are loaded in-memory. Do not use for large files.
+     * Large files will be copied to a temp file.
+     * For large String content, consider the post(File) method.
      *
      * @param field values must be String or File.
      */
@@ -53,22 +56,47 @@ public class EepPost extends EepGet {
         if (fields.isEmpty())
             throw new IllegalArgumentException();
         boolean multipart = false;
+        long sz = 0;
         for (Object o : fields.values()) {
             if (o instanceof File) {
+                sz += ((File) o).length();
                 multipart = true;
                 break;
             }
         }
         if (multipart) {
             String sep = multipart ? getSeparator() : null;
-            ByteArrayOutputStream out = new ByteArrayOutputStream(4096);
+            boolean useTmp = sz > 10000;
+            File tmp = null;
+            OutputStream out = null;
+            ByteArrayOutputStream baos = null;
             try {
+                if (useTmp) {
+                    tmp = new File(_context.getTempDir(), "eeppost-" + _context.random().nextLong() + ".dat");
+                    out = new FileOutputStream(tmp);
+                    if (_log.shouldDebug())
+                        _log.debug("Estimated size: " + sz + ", using temp file " + tmp);
+                } else {
+                    baos = new ByteArrayOutputStream(4096);
+                }
                 sendFields(out, sep, fields);
+                if (useTmp)
+                    out.close();
             } catch (IOException ioe) {
+                try { out.close(); } catch (IOException ioe2) {}
+                if (tmp != null)
+                    tmp.delete();
                 return false;
             }
             String type = "multipart/form-data, boundary=" + sep;
-            return post(type, out.toByteArray(), headerTimeout, totalTimeout, inactivityTimeout);
+            boolean rv;
+            if (tmp != null) {
+                rv = post(type, tmp, headerTimeout, totalTimeout, inactivityTimeout);
+                tmp.delete();
+            } else {
+                rv = post(type, baos.toByteArray(), headerTimeout, totalTimeout, inactivityTimeout);
+            }
+            return rv;
         } else {
             StringBuilder out = new StringBuilder(2048);
             sendFields(out, fields);
@@ -77,6 +105,9 @@ public class EepPost extends EepGet {
         }
     }
 
+    /**
+     *  In-memory, not for large POSTs
+     */
     public boolean post(String contentType, String data, long headerTimeout, long totalTimeout, long inactivityTimeout) {
         if (data.length() == 0)
             throw new IllegalArgumentException();
@@ -84,6 +115,9 @@ public class EepPost extends EepGet {
         return super.fetch(headerTimeout, totalTimeout, inactivityTimeout);
     }
 
+    /**
+     *  In-memory, not for large POSTs
+     */
     public boolean post(String contentType, byte[] data, long headerTimeout, long totalTimeout, long inactivityTimeout) {
         if (data.length == 0)
             throw new IllegalArgumentException();
@@ -91,6 +125,16 @@ public class EepPost extends EepGet {
         return super.fetch(headerTimeout, totalTimeout, inactivityTimeout);
     }
 
+    /**
+     *  For large POSTs
+     */
+    public boolean post(String contentType, File data, long headerTimeout, long totalTimeout, long inactivityTimeout) {
+        if (!data.isFile() || data.length() == 0)
+            throw new IllegalArgumentException();
+        setPostData(contentType, data);
+        return super.fetch(headerTimeout, totalTimeout, inactivityTimeout);
+    }
+
     /**
      * @throws UnsupportedOperationException always
      */
@@ -347,7 +391,7 @@ public class EepPost extends EepGet {
                            "        [-t headerTimeout]  (default 45 sec)\n" +
                            "        [-u inactivityTimeout]  (default 60 sec)\n" +
                            "        [-w totalTimeout]  (default unlimited)\n" +
-                           "        [-u username] [-x password] url\n" +
+                           "        [-u proxyUsername] [-x proxyPassword] url\n" +
                            "        (use -c or -p :0 for no proxy)");
     }