2005-11-19 jrandom

* Implemented a trivial pure java PMTU backoff strategy, switching between
      a 608 byte MTU and a 1350 byte MTU, depending upon retransmission rates.
    * Fixed new user registration in Syndie (thanks Complication!)

apps/syndie/java/src/net/i2p/syndie/web/SwitchServlet.java

@@ -34,8 +34,9 @@ public class SwitchServlet extends BaseServlet {
                   "<input type=\"submit\" name=\"action\" value=\"Logout\" /></td></tr>\n" +
                   "</form>\n" +
                   "<tr><td colspan=\"3\"><hr /></td></tr>\n" +
-                  "<form action=\"" + ThreadedHTMLRenderer.buildProfileURL(null) + "\" method=\"POST\">\n" +
+                  "<form action=\"" + ThreadedHTMLRenderer.buildProfileURL(null) + "\" method=\"POST\">\n");
-                  "<tr><td colspan=\"3\"><b>Register a new account</b></td></tr>\n" +
+        writeAuthActionFields(out);
+        out.write("<tr><td colspan=\"3\"><b>Register a new account</b></td></tr>\n" +
                   "<tr><td colspan=\"3\">Login: <input type=\"text\" name=\"login\" /> (only known locally)</td></tr>\n" +
                   "<tr><td colspan=\"3\">Password: <input type=\"password\" name=\"password\" /></td></tr>\n" +
                   "<tr><td colspan=\"3\">Public name: <input type=\"text\" name=\"accountName\" /></td></tr>\n" +

history.txt

@@ -1,4 +1,9 @@
-$Id: history.txt,v 1.323 2005/11/16 06:50:56 jrandom Exp $
+$Id: history.txt,v 1.324 2005/11/17 03:23:46 jrandom Exp $
+
+2005-11-19  jrandom
+    * Implemented a trivial pure java PMTU backoff strategy, switching between
+      a 608 byte MTU and a 1350 byte MTU, depending upon retransmission rates.
+    * Fixed new user registration in Syndie (thanks Complication!)
 
 2005-11-17  jrandom
     * More cautious file handling in Syndie

router/java/src/net/i2p/router/OutNetMessage.java

@@ -35,6 +35,7 @@ public class OutNetMessage {
     private I2NPMessage _message;
     /** cached message class name, for use after we discard the message */
     private String _messageType;
+    private int _messageTypeId;
     /** cached message ID, for use after we discard the message */
     private long _messageId;
     private long _messageSize;
@@ -145,11 +146,13 @@ public class OutNetMessage {
         _message = msg;
         if (msg != null) {
             _messageType = msg.getClass().getName();
+            _messageTypeId = msg.getType();
             _messageId = msg.getUniqueId();
         }
     }
     
     public String getMessageType() { return _messageType; }
+    public int getMessageTypeId() { return _messageTypeId; }
     public long getMessageId() { return _messageId; }
     
     public long getMessageSize() {

router/java/src/net/i2p/router/RouterVersion.java

@@ -15,9 +15,9 @@ import net.i2p.CoreVersion;
  *
  */
 public class RouterVersion {
-    public final static String ID = "$Revision: 1.291 $ $Date: 2005/11/16 06:50:57 $";
+    public final static String ID = "$Revision: 1.292 $ $Date: 2005/11/17 03:23:46 $";
     public final static String VERSION = "0.6.1.5";
-    public final static long BUILD = 2;
+    public final static long BUILD = 3;
     public static void main(String args[]) {
         System.out.println("I2P Router version: " + VERSION + "-" + BUILD);
         System.out.println("Router ID: " + RouterVersion.ID);

router/java/src/net/i2p/router/transport/udp/ACKSender.java

@@ -108,6 +108,9 @@ public class ACKSender implements Runnable {
                     //_context.statManager().getStatLog().addData(peer.getRemoteHostId().toString(), "udp.peer.sendACKCount", ackBitfields.size(), 0);
                     UDPPacket ack = _builder.buildACK(peer, ackBitfields);
                     ack.markType(1);
+                    ack.setFragmentCount(-1);
+                    ack.setMessageType(42);
+                    
                     if (_log.shouldLog(Log.INFO))
                         _log.info("Sending ACK for " + ackBitfields);
                     boolean ok = peer.allocateSendingBytes(ack.getPacket().getLength(), true);

router/java/src/net/i2p/router/transport/udp/OutboundMessageFragments.java

@@ -480,10 +480,17 @@ public class OutboundMessageFragments {
             int sparseCount = 0;
             UDPPacket rv[] = new UDPPacket[fragments]; //sparse
             for (int i = 0; i < fragments; i++) {
-                if (state.needsSending(i))
+                if (state.needsSending(i)) {
                     rv[i] = _builder.buildPacket(state, i, peer, remaining, partialACKBitfields);
-                else
+                    rv[i].setFragmentCount(fragments);
+                    OutNetMessage msg = state.getMessage();
+                    if (msg != null)
+                        rv[i].setMessageType(msg.getMessageTypeId());
+                    else
+                        rv[i].setMessageType(-1);
+                } else {
                     sparseCount++;
+                }
             }
             if (sparseCount > 0)
                 remaining.clear();

router/java/src/net/i2p/router/transport/udp/PeerState.java

@@ -138,6 +138,8 @@ public class PeerState {
     private int _mtu;
     /** when did we last check the MTU? */
     private long _mtuLastChecked;
+    private long _mtuIncreases;
+    private long _mtuDecreases;
     /** current round trip time estimate */
     private volatile int _rtt;
     /** smoothed mean deviation in the rtt */
@@ -177,9 +179,19 @@ public class PeerState {
      * for UDP, and 8 for IP, giving us 596.  round up to mod 16, giving a total
      * of 608
      */
-    private static final int DEFAULT_MTU = 608;//600; //1500;
+    private static final int MIN_MTU = 608;//600; //1500;
+    private static final int DEFAULT_MTU = MIN_MTU;
+    /* 
+     * based on measurements, 1350 fits nearly all reasonably small I2NP messages
+     * (larger I2NP messages may be up to 1900B-4500B, which isn't going to fit
+     * into a live network MTU anyway)
+     */
+    private static final int LARGE_MTU = 1350;
+    
     private static final int MIN_RTO = 500 + ACKSender.ACK_FREQUENCY;
     private static final int MAX_RTO = 1200; // 5000;
+    /** override the default MTU */
+    private static final String PROP_DEFAULT_MTU = "i2np.udp.mtu";
     
     public PeerState(I2PAppContext ctx) {
         _context = ctx;
@@ -214,7 +226,7 @@ public class PeerState {
         _remoteRequiresIntroduction = false;
         _weRelayToThemAs = 0;
         _theyRelayToUsAs = 0;
-        _mtu = DEFAULT_MTU;
+        _mtu = getDefaultMTU();
         _mtuLastChecked = -1;
         _lastACKSend = -1;
         _rtt = 1000;
@@ -234,6 +246,20 @@ public class PeerState {
         _context.statManager().createRateStat("udp.sendACKPartial", "Number of partial ACKs sent (duration == number of full ACKs in that ack packet)", "udp", new long[] { 60*1000, 10*60*1000, 60*60*1000, 24*60*60*1000 });
         _context.statManager().createRateStat("udp.sendBps", "How fast we are transmitting when a packet is acked", "udp", new long[] { 60*1000, 10*60*1000, 60*60*1000, 24*60*60*1000 });
         _context.statManager().createRateStat("udp.receiveBps", "How fast we are receiving when a packet is fully received (at most one per second)", "udp", new long[] { 60*1000, 10*60*1000, 60*60*1000, 24*60*60*1000 });
+        _context.statManager().createRateStat("udp.mtuIncrease", "How many retransmissions have there been to the peer when the MTU was increased (period is total packets transmitted)", "udp", new long[] { 60*1000, 10*60*1000, 60*60*1000, 24*60*60*1000 });
+        _context.statManager().createRateStat("udp.mtuDecrease", "How many retransmissions have there been to the peer when the MTU was decreased (period is total packets transmitted)", "udp", new long[] { 60*1000, 10*60*1000, 60*60*1000, 24*60*60*1000 });
+    }
+    
+    private int getDefaultMTU() {
+        String mtu = _context.getProperty(PROP_DEFAULT_MTU);
+        if (mtu != null) {
+            try {
+                return Integer.valueOf(mtu).intValue();
+            } catch (NumberFormatException nfe) {
+                // ignore
+            }
+        }
+        return DEFAULT_MTU;
     }
     
     /**
@@ -328,6 +354,8 @@ public class PeerState {
     public int getMTU() { return _mtu; }
     /** when did we last check the MTU? */
     public long getMTULastChecked() { return _mtuLastChecked; }
+    public long getMTUIncreases() { return _mtuIncreases; }
+    public long getMTUDecreases() { return _mtuDecreases; }
     
     
     /**
@@ -702,8 +730,16 @@ public class PeerState {
         }
         
         _messagesSent++;
-        if (numSends < 2)
+        if (numSends < 2) {
             recalculateTimeouts(lifetime);
+            if (_mtu <= MIN_MTU) {
+                if (_context.random().nextInt((int)_mtuDecreases) <= 0) {
+                    _context.statManager().addRateData("udp.mtuIncrease", _packetsRetransmitted, _packetsTransmitted);
+                    _mtu = LARGE_MTU;
+                    _mtuIncreases++;
+                }
+            }
+        }
         else if (_log.shouldLog(Log.WARN))
             _log.warn("acked after numSends=" + numSends + " w/ lifetime=" + lifetime + " and size=" + bytesACKed);
         
@@ -731,6 +767,17 @@ public class PeerState {
             _rto = MAX_RTO;
     }
     
+    private void reduceMTU() {
+        if (_mtu > MIN_MTU) {
+            double retransPct = (double)_packetsRetransmitted/(double)_packetsTransmitted;
+            if (retransPct >= 0.05) { // should we go for lower?
+                _context.statManager().addRateData("udp.mtuDecrease", _packetsRetransmitted, _packetsTransmitted);
+                _mtu = MIN_MTU;
+                _mtuDecreases++;
+            }
+        }
+    }
+    
     /** we are resending a packet, so lets jack up the rto */
     public void messageRetransmitted(int packets) { 
         long now = _context.clock().now();
@@ -745,6 +792,7 @@ public class PeerState {
         }
         congestionOccurred();
         _context.statManager().addRateData("udp.congestedRTO", _rto, _rttDeviation);
+        reduceMTU();
         //_rto *= 2; 
     }
     public void packetsTransmitted(int packets) { 

router/java/src/net/i2p/router/transport/udp/UDPEndpoint.java

@@ -74,10 +74,18 @@ public class UDPEndpoint {
      *
      * @return number of packets in the send queue
      */
-    public int send(UDPPacket packet) { return _sender.add(packet); }
+    public int send(UDPPacket packet) { 
+        if (_sender == null)
+            return 0;
+        return _sender.add(packet); 
+    }
     
     /**
      * Blocking call to receive the next inbound UDP packet from any peer.
      */
-    public UDPPacket receive() { return _receiver.receiveNext(); }
+    public UDPPacket receive() { 
+        if (_receiver == null)
+            return null;
+        return _receiver.receiveNext(); 
+    }
 }

router/java/src/net/i2p/router/transport/udp/UDPPacket.java

@@ -117,6 +117,13 @@ public class UDPPacket {
      */
     public int getMarkedType() { verifyNotReleased(); return _markedType; }
     
+    private int _messageType;
+    private int _fragmentCount;
+    int getMessageType() { return _messageType; }
+    void setMessageType(int type) { _messageType = type; }
+    int getFragmentCount() { return _fragmentCount; }
+    void setFragmentCount(int count) { _fragmentCount = count; }
+    
     public RemoteHostId getRemoteHost() {
         if (_remoteHost == null)
             _remoteHost = new RemoteHostId(_packet.getAddress().getAddress(), _packet.getPort());

router/java/src/net/i2p/router/transport/udp/UDPSender.java

@@ -41,6 +41,19 @@ public class UDPSender {
         _context.statManager().createRateStat("udp.sendBWThrottleTime", "How long the send is blocked by the bandwidth throttle", "udp", new long[] { 60*1000, 10*60*1000, 60*60*1000 });
         _context.statManager().createRateStat("udp.sendACKTime", "How long an ACK packet is blocked for (duration == lifetime)", "udp", new long[] { 60*1000, 10*60*1000, 60*60*1000 });
         _context.statManager().createRateStat("udp.sendException", "How frequently we fail to send a packet (likely due to a windows exception)", "udp", new long[] { 60*1000, 10*60*1000, 60*60*1000 });
+
+        _context.statManager().createRateStat("udp.sendPacketSize.1", "db store message size", "udp", new long[] { 60*1000, 5*60*1000, 30*60*1000 });
+        _context.statManager().createRateStat("udp.sendPacketSize.2", "db lookup message size", "udp", new long[] { 60*1000, 5*60*1000, 30*60*1000 });
+        _context.statManager().createRateStat("udp.sendPacketSize.3", "db search reply message size", "udp", new long[] { 60*1000, 5*60*1000, 30*60*1000 });
+        _context.statManager().createRateStat("udp.sendPacketSize.6", "tunnel create message size", "udp", new long[] { 60*1000, 5*60*1000, 30*60*1000 });
+        _context.statManager().createRateStat("udp.sendPacketSize.7", "tunnel create status message size", "udp", new long[] { 60*1000, 5*60*1000, 30*60*1000 });
+        _context.statManager().createRateStat("udp.sendPacketSize.10", "delivery status message size", "udp", new long[] { 60*1000, 5*60*1000, 30*60*1000 });
+        _context.statManager().createRateStat("udp.sendPacketSize.11", "garlic message size", "udp", new long[] { 60*1000, 5*60*1000, 30*60*1000 });
+        _context.statManager().createRateStat("udp.sendPacketSize.16", "date message size", "udp", new long[] { 60*1000, 5*60*1000, 30*60*1000 });
+        _context.statManager().createRateStat("udp.sendPacketSize.18", "tunnel data message size", "udp", new long[] { 60*1000, 5*60*1000, 30*60*1000 });
+        _context.statManager().createRateStat("udp.sendPacketSize.19", "tunnel gateway message size", "udp", new long[] { 60*1000, 5*60*1000, 30*60*1000 });
+        _context.statManager().createRateStat("udp.sendPacketSize.20", "data message size", "udp", new long[] { 60*1000, 5*60*1000, 30*60*1000 });
+        _context.statManager().createRateStat("udp.sendPacketSize.42", "ack-only packet size", "udp", new long[] { 60*1000, 5*60*1000, 30*60*1000 });
     }
     
     public void startup() {
@@ -155,6 +168,8 @@ public class UDPSender {
                         //_log.debug("Sending packet: (size="+size + "/"+size2 +")\nraw: " + Base64.encode(packet.getPacket().getData(), 0, size));
                     }
                     
+                    _context.statManager().addRateData("udp.sendPacketSize." + packet.getMessageType(), size, packet.getFragmentCount());
+                    
                     //packet.getPacket().setLength(size);
                     try {
                         long before = _context.clock().now();

router/java/src/net/i2p/router/transport/udp/UDPTransport.java

@@ -935,7 +935,7 @@ public class UDPTransport extends TransportImpl implements TimedWeightedPriority
         buf.append("     <td><b><a href=\"#def.up\">up</a></b></td><td><b><a href=\"#def.skew\">skew</a></b></td>\n");
         buf.append("     <td><b><a href=\"#def.cwnd\">cwnd</a></b></td><td><b><a href=\"#def.ssthresh\">ssthresh</a></b></td>\n");
         buf.append("     <td><b><a href=\"#def.rtt\">rtt</a></b></td><td><b><a href=\"#def.dev\">dev</a></b></td><td><b><a href=\"#def.rto\">rto</a></b></td>\n");
-        buf.append("     <td><b><a href=\"#def.send\">send</a></b></td><td><b><a href=\"#def.recv\">recv</a></b></td>\n");
+        buf.append("     <td><b><a href=\"#def.mtu\">mtu</a></b></td><td><b><a href=\"#def.send\">send</a></b></td><td><b><a href=\"#def.recv\">recv</a></b></td>\n");
         buf.append("     <td><b><a href=\"#def.resent\">resent</a></b></td><td><b><a href=\"#def.dupRecv\">dupRecv</a></b></td>\n");
         buf.append(" </tr>\n");
         out.write(buf.toString());
@@ -1062,6 +1062,12 @@ public class UDPTransport extends TransportImpl implements TimedWeightedPriority
             buf.append("<td valign=\"top\" ><code>");
             buf.append(rto);
             buf.append("</code></td>");
+            
+            buf.append("<td valign=\"top\" ><code>");
+            buf.append(peer.getMTU()).append('/');
+            buf.append(peer.getMTUIncreases()).append('/');
+            buf.append(peer.getMTUDecreases());
+            buf.append("</code></td>");
         
             long sent = peer.getPacketsTransmitted();
             long recv = peer.getPacketsReceived();
@@ -1113,7 +1119,7 @@ public class UDPTransport extends TransportImpl implements TimedWeightedPriority
             numPeers++;
         }
         
-        buf.append("<tr><td colspan=\"14\"><hr /></td></tr>\n");
+        buf.append("<tr><td colspan=\"15\"><hr /></td></tr>\n");
         buf.append(" <tr><td colspan=\"2\"><b>Total</b></td>");
         buf.append("     <td>");
         buf.append(formatKBps(bpsIn)).append("KBps/").append(formatKBps(bpsOut));
@@ -1127,12 +1133,12 @@ public class UDPTransport extends TransportImpl implements TimedWeightedPriority
         buf.append(numPeers > 0 ? rttTotal/numPeers : 0);
         buf.append("</td><td>&nbsp;</td><td>");
         buf.append(numPeers > 0 ? rtoTotal/numPeers : 0);
-        buf.append("</td>\n     <td>");
+        buf.append("</td>\n     <td>&nbsp;</td><td>");
         buf.append(sendTotal).append("</td><td>").append(recvTotal).append("</td>\n");
         buf.append("     <td>").append(resentTotal);
         buf.append("</td><td>").append(dupRecvTotal).append("</td>\n");
         buf.append(" </tr>\n");
-        buf.append("<tr><td colspan=\"14\" valign=\"top\" align=\"left\">");
+        buf.append("<tr><td colspan=\"15\" valign=\"top\" align=\"left\">");
         long bytesTransmitted = _context.bandwidthLimiter().getTotalAllocatedOutboundBytes();
         double averagePacketSize = _context.statManager().getRate("udp.sendPacketSize").getLifetimeAverageValue();
         // lifetime value, not just the retransmitted packets of current connections
@@ -1161,7 +1167,7 @@ public class UDPTransport extends TransportImpl implements TimedWeightedPriority
         }
     }
     
-    private static final String KEY = "<tr><td colspan=\"14\" valign=\"top\" align=\"left\">" +
+    private static final String KEY = "<tr><td colspan=\"15\" valign=\"top\" align=\"left\">" +
         "<b id=\"def.peer\">peer</b>: the remote peer (&lt; means they offer to introduce us, &gt; means we offer to introduce them)<br />\n" +
         "<b id=\"def.idle\">idle</b>: the idle time is how long since a packet has been received or sent<br />\n" +
         "<b id=\"def.rate\">in/out</b>: the rates show a smoothed inbound and outbound transfer rate (KBytes per second)<br />\n" +
@@ -1172,6 +1178,7 @@ public class UDPTransport extends TransportImpl implements TimedWeightedPriority
         "<b id=\"def.rtt\">rtt</b>: the round trip time is how long it takes to get an acknowledgement of a packet<br />\n" +
         "<b id=\"def.dev\">dev</b>: the standard deviation of the round trip time, to help control the retransmit timeout<br />\n" +
         "<b id=\"def.rto\">rto</b>: the retransmit timeout controls how frequently an unacknowledged packet will be retransmitted<br />\n" +
+        "<b id=\"def.mtu\">mtu</b>: current sending packet size/number of times it increased/number of times it decreased<br />\n" +
         "<b id=\"def.send\">send</b>: the number of packets sent to the peer<br />\n" +
         "<b id=\"def.recv\">recv</b>: the number of packets received from the peer<br />\n" +
         "<b id=\"def.resent\">resent</b>: the number of packets retransmitted to the peer<br />\n" +