* Reachability:

- Track unreachable peers persistently
        (i.e. separately from shitlist, and not cleared when they contact us)
      - Exclude detected unreachable peers from inbound tunnels
      - Exclude detected unreachable peers from selected leases
      - Exclude detected unreachable floodfill peers from lookups
      - Show unreachable status on profiles.jsp

history.txt

@@ -1,3 +1,12 @@
+2008-04-17 zzz
+    * Reachability:
+      - Track unreachable peers persistently
+        (i.e. separately from shitlist, and not cleared when they contact us)
+      - Exclude detected unreachable peers from inbound tunnels
+      - Exclude detected unreachable peers from selected leases
+      - Exclude detected unreachable floodfill peers from lookups
+      - Show unreachable status on profiles.jsp
+
 2008-04-16 zzz
     * SSU/Reachability:
       - Extend shitlist time from 4-8m to 40-60m

router/java/src/net/i2p/router/CommSystemFacade.java

@@ -54,6 +54,7 @@ public abstract class CommSystemFacade implements Service {
     public short getReachabilityStatus() { return STATUS_OK; }
     public void recheckReachability() {}
     public boolean isBacklogged(Hash dest) { return false; }
+    public boolean wasUnreachable(Hash dest) { return false; }
     
     /** 
      * Tell other transports our address changed

router/java/src/net/i2p/router/RouterVersion.java

@@ -17,7 +17,7 @@ import net.i2p.CoreVersion;
 public class RouterVersion {
     public final static String ID = "$Revision: 1.548 $ $Date: 2008-02-10 15:00:00 $";
     public final static String VERSION = "0.6.1.32";
-    public final static long BUILD = 18;
+    public final static long BUILD = 19;
     public static void main(String args[]) {
         System.out.println("I2P Router version: " + VERSION + "-" + BUILD);
         System.out.println("Router ID: " + RouterVersion.ID);

router/java/src/net/i2p/router/message/OutboundClientMessageOneShotJob.java

@@ -319,15 +319,29 @@ public class OutboundClientMessageOneShotJob extends JobImpl {
             _lease = (Lease)orderedLeases.get(orderedLeases.firstKey());
         } else {
 ****/
+
+        // Avoid a lease on a gateway we think is unreachable, if possible
+        for (int i = 0; i < _leaseSet.getLeaseCount(); i++) {
+            Lease l = _leaseSet.getLease(i);
+            if (!getContext().commSystem().wasUnreachable(l.getGateway())) {
+                _lease = l;
+                break;
+            }
+            if (_log.shouldLog(Log.WARN))
+                _log.warn(getJobId() + ": Skipping unreachable (by us) gateway " + l.getGateway()); 
+        }
+        if (_lease == null) {
             _lease = (Lease)leases.get(0);
-//      }
+            if (_log.shouldLog(Log.WARN))
+                _log.warn(getJobId() + ": All leases are unreachable (by us) for " + _toString); 
+        }
 /*** removed until we fix SSU reachability
         synchronized (_leaseCache) {
             _leaseCache.put(_to, _lease);
         }
-***/
         if (_log.shouldLog(Log.WARN))
             _log.warn("Added to cache - lease for " + _toString); 
+***/
         return true;
     }
 

router/java/src/net/i2p/router/networkdb/kademlia/FloodOnlySearchJob.java

@@ -76,7 +76,7 @@ class FloodOnlySearchJob extends FloodSearchJob {
         // We need to randomize our ff selection, else we stay with the same ones since
         // getFloodfillPeers() is sorted by closest distance. Always using the same
         // ones didn't help reliability.
-        // Also, query the unheard-from, unprofiled, failing and shitlisted ones last.
+        // Also, query the unheard-from, unprofiled, failing, unreachable and shitlisted ones last.
         // We should hear from floodfills pretty frequently so set a 30m time limit.
         // If unprofiled we haven't talked to them in a long time.
         // We aren't contacting the peer directly, so shitlist doesn't strictly matter,
@@ -90,14 +90,15 @@ class FloodOnlySearchJob extends FloodSearchJob {
                  Hash peer = (Hash)floodfillPeers.get(i);
                  PeerProfile profile = getContext().profileOrganizer().getProfile(peer);
                  if (profile == null || profile.getLastHeardFrom() < before ||
-                     profile.getIsFailing() || getContext().shitlist().isShitlisted(peer)) {
+                     profile.getIsFailing() || getContext().shitlist().isShitlisted(peer) ||
+                     getContext().commSystem().wasUnreachable(peer)) {
                      failcount++;
                      ffp.add(peer);
                  } else
                      ffp.add(0, peer);
             }
             if (_log.shouldLog(Log.INFO) && failcount > 0)
-                _log.info(getJobId() + ": " + failcount + " of " + floodfillPeers.size() + " floodfills are not heard from, unprofiled, failing or shitlisted");
+                _log.info(getJobId() + ": " + failcount + " of " + floodfillPeers.size() + " floodfills are not heard from, unprofiled, failing, unreachable or shitlisted");
             floodfillPeers = ffp;
         }
 

router/java/src/net/i2p/router/peermanager/ProfileOrganizer.java

@@ -424,6 +424,24 @@ public class ProfileOrganizer {
         return;
     }
     
+    public List selectPeersLocallyUnreachable() { 
+        List n;
+        int count;
+        synchronized (_reorganizeLock) {
+            count = _notFailingPeers.size();
+            n = new ArrayList(_notFailingPeers.keySet());
+        }
+        List l = new ArrayList(count / 4);
+        for (Iterator iter = n.iterator(); iter.hasNext(); ) {
+            Hash peer = (Hash)iter.next();
+            if (_context.commSystem().wasUnreachable(peer))
+                l.add(peer);
+        }
+        if (_log.shouldLog(Log.INFO))
+            _log.info("Unreachable: " + l);
+        return l;
+    }
+
     /**
      * Find the hashes for all peers we are actively profiling
      *

router/java/src/net/i2p/router/peermanager/ProfileOrganizerRenderer.java

@@ -137,6 +137,7 @@ class ProfileOrganizerRenderer {
             buf.append("<td>");
             if (_context.shitlist().isShitlisted(peer)) buf.append("Shitlist");
             if (prof.getIsFailing()) buf.append(" Failing");
+            if (_context.commSystem().wasUnreachable(peer)) buf.append(" Unreachable");
             buf.append("&nbsp</td>");
             //buf.append("<td><a href=\"/profile/").append(prof.getPeer().toBase64().substring(0, 32)).append("\">profile.txt</a> ");
             //buf.append("    <a href=\"#").append(prof.getPeer().toBase64().substring(0, 32)).append("\">netDb</a></td>");

router/java/src/net/i2p/router/transport/CommSystemFacadeImpl.java

@@ -125,6 +125,10 @@ public class CommSystemFacadeImpl extends CommSystemFacade {
         return _manager.isBacklogged(dest); 
     }
     
+    public boolean wasUnreachable(Hash dest) { 
+        return _manager.wasUnreachable(dest); 
+    }
+    
     public List getMostRecentErrorMessages() { 
         return _manager.getMostRecentErrorMessages(); 
     }

router/java/src/net/i2p/router/transport/Transport.java

@@ -48,6 +48,7 @@ public interface Transport {
     public short getReachabilityStatus();
     public void recheckReachability();
     public boolean isBacklogged(Hash dest);
+    public boolean wasUnreachable(Hash dest);
     
     public boolean isUnreachable(Hash peer);
 }

router/java/src/net/i2p/router/transport/TransportImpl.java

@@ -15,6 +15,7 @@ import java.util.*;
 import net.i2p.data.Hash;
 import net.i2p.data.RouterAddress;
 import net.i2p.data.RouterIdentity;
+import net.i2p.data.RouterInfo;
 import net.i2p.data.i2np.I2NPMessage;
 import net.i2p.router.CommSystemFacade;
 import net.i2p.router.Job;
@@ -36,6 +37,7 @@ public abstract class TransportImpl implements Transport {
     protected RouterContext _context;
     /** map from routerIdentHash to timestamp (Long) that the peer was last unreachable */
     private Map _unreachableEntries;
+    private Set _wasUnreachableEntries;
 
     /**
      * Initialize the new transport
@@ -54,6 +56,7 @@ public abstract class TransportImpl implements Transport {
         _context.statManager().createRateStat("transport.expiredOnQueueLifetime", "How long a message that expires on our outbound queue is processed", "Transport", new long[] { 60*1000l, 10*60*1000l, 60*60*1000l, 24*60*60*1000l } );
         _sendPool = new ArrayList(16);
         _unreachableEntries = new HashMap(16);
+        _wasUnreachableEntries = new HashSet(16);
         _currentAddress = null;
     }
     
@@ -399,19 +402,23 @@ public abstract class TransportImpl implements Transport {
         }
     }
     /** called when we can't reach a peer */
+    /** This isn't very useful since it is cleared when they contact us */
     public void markUnreachable(Hash peer) {
         long now = _context.clock().now();
         synchronized (_unreachableEntries) {
             _unreachableEntries.put(peer, new Long(now));
         }
+        markWasUnreachable(peer, true);
     }
     /** called when we establish a peer connection (outbound or inbound) */
-    public void markReachable(Hash peer) {
+    public void markReachable(Hash peer, boolean isInbound) {
         // if *some* transport can reach them, then we shouldn't shitlist 'em
         _context.shitlist().unshitlistRouter(peer);
         synchronized (_unreachableEntries) {
             _unreachableEntries.remove(peer);
         }
+        if (!isInbound)
+            markWasUnreachable(peer, false);
     }
     private class CleanupUnreachable extends JobImpl {
         public CleanupUnreachable(RouterContext ctx) {
@@ -432,6 +439,34 @@ public abstract class TransportImpl implements Transport {
         }
     }
 
+    /**
+     * Was the peer UNreachable (outbound only) the last time we tried it?
+     * This is NOT reset if the peer contacts us and it is never expired.
+     */
+    public boolean wasUnreachable(Hash peer) {
+        synchronized (_wasUnreachableEntries) {
+            if (_wasUnreachableEntries.contains(peer))
+                return true;
+        }
+        RouterInfo ri = _context.netDb().lookupRouterInfoLocally(peer);
+        if (ri == null)
+            return false;
+        return null == ri.getTargetAddress(this.getStyle());
+    }
+    /**
+     * Maintain the WasUnreachable list
+     */
+    public void markWasUnreachable(Hash peer, boolean yes) {
+        synchronized (_wasUnreachableEntries) {
+            if (yes)
+                _wasUnreachableEntries.add(peer);
+            else
+                _wasUnreachableEntries.remove(peer);
+        }
+        if (_log.shouldLog(Log.WARN))
+            _log.warn(this.getStyle() + " setting wasUnreachable to " + yes + " for " + peer);
+    }
+
     public static boolean isPubliclyRoutable(byte addr[]) {
         if (addr.length == 4) {
             if ((addr[0]&0xFF) == 127) return false;

router/java/src/net/i2p/router/transport/TransportManager.java

@@ -202,6 +202,20 @@ public class TransportManager implements TransportEventListener {
         return false;
     }    
     
+    /**
+     * Was the peer UNreachable (outbound only) on any transport,
+     * based on the last time we tried it for each transport?
+     * This is NOT reset if the peer contacts us.
+     */
+    public boolean wasUnreachable(Hash dest) {
+        for (int i = 0; i < _transports.size(); i++) {
+            Transport t = (Transport)_transports.get(i);
+            if (!t.wasUnreachable(dest))
+                return false;
+        }
+        return true;
+    }    
+    
     Map getAddresses() {
         Map rv = new HashMap(_transports.size());
         for (int i = 0; i < _transports.size(); i++) {

router/java/src/net/i2p/router/transport/ntcp/EstablishState.java

@@ -351,7 +351,7 @@ public class EstablishState {
                 long diff = 1000*Math.abs(_tsA-_tsB);
                 if (diff >= Router.CLOCK_FUDGE_FACTOR) {
                     _context.statManager().addRateData("ntcp.invalidOutboundSkew", diff, 0);
-                    _transport.markReachable(_con.getRemotePeer().calculateHash());
+                    _transport.markReachable(_con.getRemotePeer().calculateHash(), false);
                     _context.shitlist().shitlistRouter(_con.getRemotePeer().calculateHash(), "Outbound clock skew of " + diff + " ms");
                     fail("Clocks too skewed (" + diff + " ms)", null, true);
                     return;
@@ -536,7 +536,7 @@ public class EstablishState {
                 long diff = 1000*Math.abs(tsA-_tsB);
                 if (diff >= Router.CLOCK_FUDGE_FACTOR) {
                     _context.statManager().addRateData("ntcp.invalidInboundSkew", diff, 0);
-                    _transport.markReachable(alice.calculateHash());
+                    _transport.markReachable(alice.calculateHash(), true);
                     _context.shitlist().shitlistRouter(alice.calculateHash(), "Clock skew of " + diff + " ms");
                     fail("Clocks too skewed (" + diff + " ms)", null, true);
                     return;

router/java/src/net/i2p/router/transport/ntcp/NTCPConnection.java

@@ -419,7 +419,7 @@ public class NTCPConnection implements FIFOBandwidthLimiter.CompleteListener {
         _established = true;
         _establishedOn = System.currentTimeMillis();
         _establishState = null;
-        _transport.markReachable(getRemotePeer().calculateHash());
+        _transport.markReachable(getRemotePeer().calculateHash(), false);
         //_context.shitlist().unshitlistRouter(getRemotePeer().calculateHash(), NTCPTransport.STYLE);
         boolean msgs = false;
         synchronized (_outbound) {

router/java/src/net/i2p/router/transport/ntcp/NTCPTransport.java

@@ -132,7 +132,7 @@ public class NTCPTransport extends TransportImpl {
     
     void inboundEstablished(NTCPConnection con) {
         _context.statManager().addRateData("ntcp.inboundEstablished", 1, 0);
-        markReachable(con.getRemotePeer().calculateHash());
+        markReachable(con.getRemotePeer().calculateHash(), true);
         //_context.shitlist().unshitlistRouter(con.getRemotePeer().calculateHash());
         NTCPConnection old = null;
         synchronized (_conLock) {

router/java/src/net/i2p/router/transport/udp/UDPTransport.java

@@ -555,7 +555,7 @@ public class UDPTransport extends TransportImpl implements TimedWeightedPriority
             _log.warn("Peer already connected: old=" + oldPeer + " new=" + peer, new Exception("dup"));
         
         _activeThrottle.unchoke(peer.getRemotePeer());
-        markReachable(peer.getRemotePeer());
+        markReachable(peer.getRemotePeer(), peer.isInbound());
         //_context.shitlist().unshitlistRouter(peer.getRemotePeer(), STYLE);
 
         if (SHOULD_FLOOD_PEERS)

router/java/src/net/i2p/router/tunnel/pool/TunnelPeerSelector.java

@@ -155,8 +155,12 @@ public abstract class TunnelPeerSelector {
         //
         // Unreachable peers at the inbound gateway is a major cause of problems.
         // Due to a bug in SSU peer testing in 0.6.1.32 and earlier, peers don't know
-        // if they are unreachable, so this won't help much. As of 0.6.1.33 we should have
-        // lots of unreachables, so enable this for now.
+        // if they are unreachable, so the netdb indication won't help much.
+        // As of 0.6.1.33 we should have lots of unreachables, so enable this for now.
+        // Also (and more effectively) exclude peers we detect are unreachable,
+        // this should be much more effective, especially on a router that has been
+        // up a few hours.
+        //
         // We could just try and exclude them as the inbound gateway but that's harder
         // (and even worse for anonymity?).
         //
@@ -168,6 +172,9 @@ public abstract class TunnelPeerSelector {
             List caps = ctx.peerManager().getPeersByCapability(Router.CAPABILITY_UNREACHABLE);
             if (caps != null)
                 peers.addAll(caps);
+            caps = ctx.profileOrganizer().selectPeersLocallyUnreachable();
+            if (caps != null)
+                peers.addAll(caps);
         }
         if (filterSlow(ctx, isInbound, isExploratory)) {
             Log log = ctx.logManager().getLog(TunnelPeerSelector.class);