From f9654661bb0df5e43b1b28eff59723a4ba3db84b Mon Sep 17 00:00:00 2001 From: zzz Date: Wed, 11 May 2011 14:40:55 +0000 Subject: [PATCH] * Use partitions of fast tier for various hops of client tunnels; minor cleanups --- history.txt | 3 + .../src/net/i2p/router/RouterVersion.java | 2 +- .../router/peermanager/ProfileOrganizer.java | 140 +++++++++++++++--- .../tunnel/pool/ClientPeerSelector.java | 53 ++++++- .../tunnel/pool/TunnelPeerSelector.java | 7 +- 5 files changed, 177 insertions(+), 28 deletions(-) diff --git a/history.txt b/history.txt index 177b44bb1..8bfc3bda9 100644 --- a/history.txt +++ b/history.txt @@ -1,3 +1,6 @@ +2011-05-11 zzz + * Use partitions of fast tier for various hops of client tunnels + 2011-05-06 zzz * Tunnels and profiles: - Increase max fast and high-cap tier sizes diff --git a/router/java/src/net/i2p/router/RouterVersion.java b/router/java/src/net/i2p/router/RouterVersion.java index 282c18b42..0725033fa 100644 --- a/router/java/src/net/i2p/router/RouterVersion.java +++ b/router/java/src/net/i2p/router/RouterVersion.java @@ -18,7 +18,7 @@ public class RouterVersion { /** deprecated */ public final static String ID = "Monotone"; public final static String VERSION = CoreVersion.VERSION; - public final static long BUILD = 2; + public final static long BUILD = 3; /** for example "-test" */ public final static String EXTRA = ""; diff --git a/router/java/src/net/i2p/router/peermanager/ProfileOrganizer.java b/router/java/src/net/i2p/router/peermanager/ProfileOrganizer.java index 93652d92e..a63dd9769 100644 --- a/router/java/src/net/i2p/router/peermanager/ProfileOrganizer.java +++ b/router/java/src/net/i2p/router/peermanager/ProfileOrganizer.java @@ -2,7 +2,6 @@ package net.i2p.router.peermanager; import java.io.IOException; import java.io.OutputStream; -import java.io.Writer; import java.net.InetAddress; import java.net.UnknownHostException; import java.text.DecimalFormat; @@ -22,6 +21,7 @@ import java.util.TreeSet; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantReadWriteLock; +import net.i2p.crypto.SHA256Generator; import net.i2p.data.Hash; import net.i2p.data.RouterAddress; import net.i2p.data.RouterInfo; @@ -78,7 +78,7 @@ public class ProfileOrganizer { public static final String PROP_MINIMUM_FAST_PEERS = "profileOrganizer.minFastPeers"; public static final int DEFAULT_MINIMUM_FAST_PEERS = 8; /** this is misnamed, it is really the max minimum number. */ - private static final int DEFAULT_MAXIMUM_FAST_PEERS = 16; + private static final int DEFAULT_MAXIMUM_FAST_PEERS = 18; private static final int ABSOLUTE_MAX_FAST_PEERS = 60; /** @@ -94,9 +94,6 @@ public class ProfileOrganizer { /** synchronized against this lock when updating the tier that peers are located in (and when fetching them from a peer) */ private final ReentrantReadWriteLock _reorganizeLock = new ReentrantReadWriteLock(true); - /** incredibly weak PRNG, just used for shuffling peers. no need to waste the real PRNG on this */ - private Random _random = new Random(); - public ProfileOrganizer(RouterContext context) { _context = context; _log = context.logManager().getLog(ProfileOrganizer.class); @@ -108,9 +105,6 @@ public class ProfileOrganizer { _notFailingPeersList = new ArrayList(256); _failingPeers = new HashMap(16); _strictCapacityOrder = new TreeSet(_comp); - _thresholdSpeedValue = 0.0d; - _thresholdCapacityValue = 0.0d; - _thresholdIntegrationValue = 0.0d; _persistenceHelper = new ProfilePersistenceHelper(_context); _context.statManager().createRateStat("peer.profileSortTime", "How long the reorg takes sorting peers", "Peers", new long[] { 10*60*1000 }); @@ -279,14 +273,25 @@ public class ProfileOrganizer { * @param howMany how many peers are desired * @param exclude set of Hashes for routers that we don't want selected * @param matches set to store the return value in - * @param mask 0-4 Number of bytes to match to determine if peers in the same IP range should - * not be in the same tunnel. 0 = disable check; 1 = /8; 2 = /16; 3 = /24; 4 = exact IP match * */ public void selectFastPeers(int howMany, Set exclude, Set matches) { selectFastPeers(howMany, exclude, matches, 0); } + /** + * Return a set of Hashes for peers that are both fast and reliable. If an insufficient + * number of peers are both fast and reliable, fall back onto high capacity peers, and if that + * doesn't contain sufficient peers, fall back onto not failing peers, and even THAT doesn't + * have sufficient peers, fall back onto failing peers. + * + * @param howMany how many peers are desired + * @param exclude set of Hashes for routers that we don't want selected + * @param matches set to store the return value in + * @param mask 0-4 Number of bytes to match to determine if peers in the same IP range should + * not be in the same tunnel. 0 = disable check; 1 = /8; 2 = /16; 3 = /24; 4 = exact IP match + * + */ public void selectFastPeers(int howMany, Set exclude, Set matches, int mask) { getReadLock(); try { @@ -303,6 +308,51 @@ public class ProfileOrganizer { return; } + /** + * Return a set of Hashes for peers that are both fast and reliable. If an insufficient + * number of peers are both fast and reliable, fall back onto high capacity peers, and if that + * doesn't contain sufficient peers, fall back onto not failing peers, and even THAT doesn't + * have sufficient peers, fall back onto failing peers. + * + * @param howMany how many peers are desired + * @param exclude set of Hashes for routers that we don't want selected + * @param matches set to store the return value in + * @param randomKey used for deterministic random partitioning into subtiers + * @param subTierMode 0 or 2-7: + *
+     *    0: no partitioning, use entire tier
+     *    2: return only from group 0 or 1
+     *    3: return only from group 2 or 3
+     *    4: return only from group 0
+     *    5: return only from group 1
+     *    6: return only from group 2
+     *    7: return only from group 3
+     *
+ */ + public void selectFastPeers(int howMany, Set exclude, Set matches, Hash randomKey, int subTierMode) { + getReadLock(); + try { + if (subTierMode > 0) { + int sz = _fastPeers.size(); + if (sz < 6 || (subTierMode >= 4 && sz < 12)) + subTierMode = 0; + } + if (subTierMode > 0) + locked_selectPeers(_fastPeers, howMany, exclude, matches, randomKey, subTierMode); + else + locked_selectPeers(_fastPeers, howMany, exclude, matches, 2); + } finally { releaseReadLock(); } + if (matches.size() < howMany) { + if (_log.shouldLog(Log.INFO)) + _log.info("selectFastPeers("+howMany+"), not enough fast (" + matches.size() + ") going on to highCap"); + selectHighCapacityPeers(howMany, exclude, matches, 2); + } else { + if (_log.shouldLog(Log.INFO)) + _log.info("selectFastPeers("+howMany+"), found enough fast (" + matches.size() + ")"); + } + return; + } + /** * Return a set of Hashes for peers that have a high capacity * @@ -343,13 +393,17 @@ public class ProfileOrganizer { /** * Return a set of Hashes for peers that are well integrated into the network. * - * @param mask 0-4 Number of bytes to match to determine if peers in the same IP range should - * not be in the same tunnel. 0 = disable check; 1 = /8; 2 = /16; 3 = /24; 4 = exact IP match */ public void selectWellIntegratedPeers(int howMany, Set exclude, Set matches) { selectWellIntegratedPeers(howMany, exclude, matches, 0); } + /** + * Return a set of Hashes for peers that are well integrated into the network. + * + * @param mask 0-4 Number of bytes to match to determine if peers in the same IP range should + * not be in the same tunnel. 0 = disable check; 1 = /8; 2 = /16; 3 = /24; 4 = exact IP match + */ public void selectWellIntegratedPeers(int howMany, Set exclude, Set matches, int mask) { getReadLock(); try { @@ -1063,10 +1117,10 @@ public class ProfileOrganizer { /** called after locking the reorganizeLock */ private PeerProfile locked_getProfile(Hash peer) { - PeerProfile cur = (PeerProfile)_notFailingPeers.get(peer); + PeerProfile cur = _notFailingPeers.get(peer); if (cur != null) return cur; - cur = (PeerProfile)_failingPeers.get(peer); + cur = _failingPeers.get(peer); return cur; } @@ -1158,7 +1212,7 @@ public class ProfileOrganizer { } /** generate an arbitrary unique value for this ip/mask (mask = 1-4) */ - private Integer maskedIP(byte[] ip, int mask) { + private static Integer maskedIP(byte[] ip, int mask) { int rv = 0; for (int i = 0; i < mask; i++) rv = (rv << 8) | (ip[i] & 0xff); @@ -1166,7 +1220,7 @@ public class ProfileOrganizer { } /** does a contain any of the elements in b? */ - private boolean containsAny(Set a, Set b) { + private static boolean containsAny(Set a, Set b) { for (Object o : b) { if (a.contains(o)) return true; @@ -1174,6 +1228,58 @@ public class ProfileOrganizer { return false; } + /** + * @param randomKey used for deterministic random partitioning into subtiers + * @param subTierMode 2-7: + *
+     *    2: return only from group 0 or 1
+     *    3: return only from group 2 or 3
+     *    4: return only from group 0
+     *    5: return only from group 1
+     *    6: return only from group 2
+     *    7: return only from group 3
+     *
+ */ + private void locked_selectPeers(Map peers, int howMany, Set toExclude, Set matches, Hash randomKey, int subTierMode) { + List all = new ArrayList(peers.keySet()); + // use RandomIterator to avoid shuffling the whole thing + for (Iterator iter = new RandomIterator(all); (matches.size() < howMany) && iter.hasNext(); ) { + Hash peer = iter.next(); + if (toExclude != null && toExclude.contains(peer)) + continue; + if (matches.contains(peer)) + continue; + if (_us.equals(peer)) + continue; + int subTier = getSubTier(peer, randomKey); + if (subTierMode >= 4) { + if (subTier != (subTierMode & 0x03)) + continue; + } else { + if ((subTier >> 1) != (subTierMode & 0x01)) + continue; + } + boolean ok = isSelectable(peer); + if (ok) + matches.add(peer); + else + matches.remove(peer); + } + } + + /** + * Implement a random, deterministic split into 4 groups that cannot be predicted by + * others. + * @return 0-3 + */ + private static int getSubTier(Hash peer, Hash randomKey) { + byte[] data = new byte[Hash.HASH_LENGTH + 4]; + System.arraycopy(peer.getData(), 0, data, 0, Hash.HASH_LENGTH); + System.arraycopy(randomKey.getData(), 0, data, Hash.HASH_LENGTH, 4); + Hash rh = SHA256Generator.getInstance().calculateHash(data); + return rh.getData()[0] & 0x03; + } + public boolean isSelectable(Hash peer) { NetworkDatabaseFacade netDb = _context.netDb(); // the CLI shouldn't depend upon the netDb @@ -1288,7 +1394,7 @@ public class ProfileOrganizer { */ protected int getMinimumFastPeers() { int def = Math.min(DEFAULT_MAXIMUM_FAST_PEERS, - (2 *_context.clientManager().listClients().size()) + DEFAULT_MINIMUM_FAST_PEERS - 2); + (6 *_context.clientManager().listClients().size()) + DEFAULT_MINIMUM_FAST_PEERS - 2); return _context.getProperty(PROP_MINIMUM_FAST_PEERS, def); } diff --git a/router/java/src/net/i2p/router/tunnel/pool/ClientPeerSelector.java b/router/java/src/net/i2p/router/tunnel/pool/ClientPeerSelector.java index f4042feee..24e6966a1 100644 --- a/router/java/src/net/i2p/router/tunnel/pool/ClientPeerSelector.java +++ b/router/java/src/net/i2p/router/tunnel/pool/ClientPeerSelector.java @@ -21,20 +21,57 @@ class ClientPeerSelector extends TunnelPeerSelector { return null; if ( (length == 0) && (settings.getLength()+settings.getLengthVariance() > 0) ) return null; - HashSet matches = new HashSet(length); + + List rv; if (length > 0) { if (shouldSelectExplicit(settings)) return selectExplicit(ctx, settings, length); + + Set exclude = getExclude(ctx, settings.isInbound(), settings.isExploratory()); + Set matches = new HashSet(length); + if (length == 1) { + ctx.profileOrganizer().selectFastPeers(length, exclude, matches, 0); + matches.remove(ctx.routerHash()); + rv = new ArrayList(matches); + } else { + // build a tunnel using 4 subtiers. + // For a 2-hop tunnel, the first hop comes from subtiers 0-1 and the last from subtiers 2-3. + // For a longer tunnels, the first hop comes from subtier 0, the middle from subtiers 2-3, and the last from subtier 1. + rv = new ArrayList(length + 1); + // OBEP or IB last hop + // group 0 or 1 if two hops, otherwise group 0 + ctx.profileOrganizer().selectFastPeers(1, exclude, matches, settings.getRandomKey(), length == 2 ? 2 : 4); + matches.remove(ctx.routerHash()); + exclude.addAll(matches); + rv.addAll(matches); + matches.clear(); + if (length > 2) { + // middle hop(s) + // group 2 or 3 + ctx.profileOrganizer().selectFastPeers(length - 2, exclude, matches, settings.getRandomKey(), 3); + matches.remove(ctx.routerHash()); + if (matches.size() > 1) { + // order the middle peers for tunnels >= 4 hops + List ordered = new ArrayList(matches); + orderPeers(ordered, settings.getRandomKey()); + rv.addAll(ordered); + } else { + rv.addAll(matches); + } + exclude.addAll(matches); + matches.clear(); + } + // IBGW or OB first hop + // group 2 or 3 if two hops, otherwise group 1 + ctx.profileOrganizer().selectFastPeers(1, exclude, matches, settings.getRandomKey(), length == 2 ? 3 : 5); + matches.remove(ctx.routerHash()); + rv.addAll(matches); + } + } else { + rv = new ArrayList(1); } - Set exclude = getExclude(ctx, settings.isInbound(), settings.isExploratory()); - ctx.profileOrganizer().selectFastPeers(length, exclude, matches, settings.getIPRestriction()); - - matches.remove(ctx.routerHash()); - ArrayList rv = new ArrayList(matches); - if (rv.size() > 1) - orderPeers(rv, settings.getRandomKey()); if (settings.isInbound()) rv.add(0, ctx.routerHash()); else diff --git a/router/java/src/net/i2p/router/tunnel/pool/TunnelPeerSelector.java b/router/java/src/net/i2p/router/tunnel/pool/TunnelPeerSelector.java index 111a1f397..25b6ed933 100644 --- a/router/java/src/net/i2p/router/tunnel/pool/TunnelPeerSelector.java +++ b/router/java/src/net/i2p/router/tunnel/pool/TunnelPeerSelector.java @@ -41,6 +41,9 @@ public abstract class TunnelPeerSelector { */ public abstract List selectPeers(RouterContext ctx, TunnelPoolSettings settings); + /** + * @return randomized number of hops 0-7, not including ourselves + */ protected int getLength(RouterContext ctx, TunnelPoolSettings settings) { int length = settings.getLength(); int override = settings.getLengthOverride(); @@ -61,8 +64,8 @@ public abstract class TunnelPeerSelector { } if (length < 0) length = 0; - else if (length > 8) // as documented in tunnel.html - length = 8; + else if (length > 7) // as documented in tunnel.html + length = 7; /* if ( (ctx.tunnelManager().getOutboundTunnelCount() <= 0) || (ctx.tunnelManager().getFreeTunnelCount() <= 0) ) {