make it work with any host charset or content charset

apps/syndie/java/src/net/i2p/syndie/Archive.java

@@ -70,6 +70,7 @@ public class Archive {
                             info.add(bi);
                         } else {
                             System.err.println("Invalid blog (but we're storing it anyway): " + bi);
+                            new Exception("foo").printStackTrace();
                             info.add(bi);
                         }
                     } catch (IOException ioe) {
@@ -104,6 +105,7 @@ public class Archive {
     public boolean storeBlogInfo(BlogInfo info) { 
         if (!info.verify(_context)) {
             System.err.println("Not storing the invalid blog " + info);
+            new Exception("foo!").printStackTrace();
             return false;
         }
         boolean isNew = true;
@@ -161,13 +163,17 @@ public class Archive {
         for (int j = 0; j < entries.length; j++) {
             try {
                 File entryDir = getEntryDir(entries[j]);
-                if (!entryDir.exists()) {
+                EntryContainer entry = null;
+                if (entryDir.exists())
+                    entry = getCachedEntry(entryDir);
+                if ( (entry == null) || (!entryDir.exists()) ) {
                     if (!extractEntry(entries[j], entryDir, info)) {
                         System.err.println("Entry " + entries[j].getPath() + " is not valid");
+                        new Exception("foo!!").printStackTrace();
                         continue;
                     }
+                    entry = getCachedEntry(entryDir);
                 }
-                EntryContainer entry = getCachedEntry(entryDir);
                 String tags[] = entry.getTags();
                 for (int t = 0; t < tags.length; t++) {
                     if (!rv.contains(tags[t])) {
@@ -202,7 +208,16 @@ public class Archive {
     }
     
     private EntryContainer getCachedEntry(File entryDir) {
-        return new CachedEntry(entryDir);
+        try {
+            return new CachedEntry(entryDir);
+        } catch (IOException ioe) {
+            ioe.printStackTrace();
+            File files[] = entryDir.listFiles();
+            for (int i = 0; i < files.length; i++)
+                files[i].delete();
+            entryDir.delete();
+            return null;
+        }
     }
     
     public EntryContainer getEntry(BlogURI uri) { return getEntry(uri, null); }
@@ -233,13 +248,16 @@ public class Archive {
                 if (blogKey == null) {
                     // no key, cache.
                     File entryDir = getEntryDir(entries[i]);
-                    if (!entryDir.exists()) {
+                    if (entryDir.exists())
+                        entry = getCachedEntry(entryDir);
+                    if ((entry == null) || !entryDir.exists()) {
                         if (!extractEntry(entries[i], entryDir, info)) {
                             System.err.println("Entry " + entries[i].getPath() + " is not valid");
+                            new Exception("foo!!!!").printStackTrace();
                             continue;
                         }
+                        entry = getCachedEntry(entryDir);
                     }
-                    entry = getCachedEntry(entryDir);
                 } else {
                     // we have an explicit key - no caching
                     entry = new EntryContainer();
@@ -247,6 +265,7 @@ public class Archive {
                     boolean ok = entry.verifySignature(_context, info);
                     if (!ok) {
                         System.err.println("Keyed entry " + entries[i].getPath() + " is not valid");
+                        new Exception("foo!!!!!!").printStackTrace();
                         continue;
                     }
 
@@ -389,8 +408,8 @@ public class Archive {
         reloadInfo();
         _index = ArchiveIndexer.index(_context, this);
         try {
-            PrintWriter out = new PrintWriter(new FileWriter(new File(_rootDir, INDEX_FILE)));
+            FileOutputStream out = new FileOutputStream(new File(_rootDir, INDEX_FILE));
-            out.println(_index.toString());
+            out.write(DataHelper.getUTF8(_index.toString()));
             out.flush();
         } catch (IOException ioe) {
             ioe.printStackTrace();

apps/syndie/java/src/net/i2p/syndie/ArchiveIndexer.java

@@ -24,7 +24,7 @@ class ArchiveIndexer {
         File headerFile = new File(rootDir, Archive.HEADER_FILE);
         if (headerFile.exists()) {
             try {
-                BufferedReader in = new BufferedReader(new InputStreamReader(new FileInputStream(headerFile)));
+                BufferedReader in = new BufferedReader(new InputStreamReader(new FileInputStream(headerFile), "UTF-8"));
                 String line = null;
                 while ( (line = in.readLine()) != null) {
                     StringTokenizer tok = new StringTokenizer(line, ":");
@@ -79,6 +79,7 @@ class ArchiveIndexer {
                 for (int t = 0; t < entryTags.length; t++) {
                     if (!tags.containsKey(entryTags[t])) {
                         tags.put(entryTags[t], new TreeMap());
+                        //System.err.println("New tag [" + entryTags[t] + "]");
                     }
                     Map entriesByTag = (Map)tags.get(entryTags[t]);
                     entriesByTag.put(new Long(0-entry.getURI().getEntryId()), entry);

apps/syndie/java/src/net/i2p/syndie/BlogManager.java

@@ -87,7 +87,7 @@ public class BlogManager {
             for (Iterator iter = _context.getPropertyNames().iterator(); iter.hasNext(); ) {
                 String name = (String)iter.next();
                 if (name.startsWith("syndie."))
-                    out.write((name + '=' + _context.getProperty(name) + '\n').getBytes());
+                    out.write(DataHelper.getUTF8(name + '=' + _context.getProperty(name) + '\n'));
             }
         } catch (IOException ioe) {
             ioe.printStackTrace();
@@ -181,14 +181,17 @@ public class BlogManager {
     }
     
     public String login(User user, String login, String pass) {
-        File userFile = new File(_userDir, Base64.encode(_context.sha().calculateHash(login.getBytes()).getData()));
+        Hash userHash = _context.sha().calculateHash(DataHelper.getUTF8(login));
+        Hash passHash = _context.sha().calculateHash(DataHelper.getUTF8(pass));
+        File userFile = new File(_userDir, Base64.encode(userHash.getData()));
         System.out.println("Attempting to login to " + login + " w/ pass = " + pass 
                            + ": file = " + userFile.getAbsolutePath() + " passHash = "
-                           + Base64.encode(_context.sha().calculateHash(pass.getBytes()).getData()));
+                           + Base64.encode(passHash.getData()));
         if (userFile.exists()) {
             try {
                 Properties props = new Properties();
-                BufferedReader in = new BufferedReader(new FileReader(userFile));
+                FileInputStream fin = new FileInputStream(userFile);
+                BufferedReader in = new BufferedReader(new InputStreamReader(fin, "UTF-8"));
                 String line = null;
                 while ( (line = in.readLine()) != null) {
                     int split = line.indexOf('=');
@@ -216,12 +219,12 @@ public class BlogManager {
     
     public void saveUser(User user) {
         if (!user.getAuthenticated()) return;
-        String userHash = Base64.encode(_context.sha().calculateHash(user.getUsername().getBytes()).getData());
+        String userHash = Base64.encode(_context.sha().calculateHash(DataHelper.getUTF8(user.getUsername())).getData());
         File userFile = new File(_userDir, userHash);
-        FileWriter out = null;
+        FileOutputStream out = null;
         try {
-            out = new FileWriter(userFile);
+            out = new FileOutputStream(userFile);
-            out.write(user.export());
+            out.write(DataHelper.getUTF8(user.export()));
         } catch (IOException ioe) {
             ioe.printStackTrace();
         } finally {
@@ -229,28 +232,36 @@ public class BlogManager {
         }
     }
     public String register(User user, String login, String password, String registrationPassword, String blogName, String blogDescription, String contactURL) {
+        System.err.println("Register [" + login + "] pass [" + password + "] name [" + blogName + "] descr [" + blogDescription + "] contact [" + contactURL + "]");
+        System.err.println("reference bad string: [" + EncodingTestGenerator.TEST_STRING + "]");
         String hashedRegistrationPassword = getRegistrationPassword();
         if (hashedRegistrationPassword != null) {
-            if (!hashedRegistrationPassword.equals(Base64.encode(_context.sha().calculateHash(registrationPassword.getBytes()).getData())))
+            try {
-                return "Invalid registration password";
+                if (!hashedRegistrationPassword.equals(Base64.encode(_context.sha().calculateHash(registrationPassword.getBytes("UTF-8")).getData())))
+                    return "Invalid registration password";
+            } catch (UnsupportedEncodingException uee) {
+                return "Error registering";
+            }
         }
-        String userHash = Base64.encode(_context.sha().calculateHash(login.getBytes()).getData());
+        String userHash = Base64.encode(_context.sha().calculateHash(DataHelper.getUTF8(login)).getData());
         File userFile = new File(_userDir, userHash);
         if (userFile.exists()) {
             return "Cannot register the login " + login + ": it already exists";
         } else {
             BlogInfo info = createBlog(blogName, blogDescription, contactURL, null);
-            String hashedPassword = Base64.encode(_context.sha().calculateHash(password.getBytes()).getData());
+            String hashedPassword = Base64.encode(_context.sha().calculateHash(DataHelper.getUTF8(password)).getData());
-            FileWriter out = null;
+            FileOutputStream out = null;
             try {
-                out = new FileWriter(userFile);
+                out = new FileOutputStream(userFile);
-                out.write("password=" + hashedPassword + "\n");
+                BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(out, "UTF-8"));
-                out.write("blog=" + Base64.encode(info.getKey().calculateHash().getData()) + "\n");
+                bw.write("password=" + hashedPassword + "\n");
-                out.write("lastid=-1\n");
+                bw.write("blog=" + Base64.encode(info.getKey().calculateHash().getData()) + "\n");
-                out.write("lastmetaedition=0\n");
+                bw.write("lastid=-1\n");
-                out.write("addressbook=userhosts-"+userHash + ".txt\n");
+                bw.write("lastmetaedition=0\n");
-                out.write("showimages=false\n");
+                bw.write("addressbook=userhosts-"+userHash + ".txt\n");
-                out.write("showexpanded=false\n");
+                bw.write("showimages=false\n");
+                bw.write("showexpanded=false\n");
+                bw.flush();
             } catch (IOException ioe) {
                 ioe.printStackTrace();
                 return "Internal error registering - " + ioe.getMessage();
@@ -297,7 +308,7 @@ public class BlogManager {
             raw.append('\n');
             if ( (entryHeaders != null) && (entryHeaders.trim().length() > 0) ) {
                 System.out.println("Entry headers: " + entryHeaders);
-                BufferedReader userHeaders = new BufferedReader(new InputStreamReader(new ByteArrayInputStream(entryHeaders.getBytes())));
+                BufferedReader userHeaders = new BufferedReader(new InputStreamReader(new ByteArrayInputStream(DataHelper.getUTF8(entryHeaders)), "UTF-8"));
                 String line = null;
                 while ( (line = userHeaders.readLine()) != null) {
                     line = line.trim();
@@ -314,7 +325,7 @@ public class BlogManager {
             raw.append('\n');
             raw.append(sml);
             
-            EntryContainer c = new EntryContainer(uri, tagList, raw.toString().getBytes());
+            EntryContainer c = new EntryContainer(uri, tagList, DataHelper.getUTF8(raw));
             if ((fileNames != null) && (fileStreams != null) && (fileNames.size() == fileStreams.size()) ) {
                 for (int i = 0; i < fileNames.size(); i++) {
                     String name = (String)fileNames.get(i);
@@ -397,14 +408,14 @@ public class BlogManager {
         if (!validateAddressSchema(schema)) return "Unsupported schema: " + HTMLRenderer.sanitizeString(schema);
         // no need to quote user/location further, as they've been sanitized
         
-        FileWriter out = null;
+        FileOutputStream out = null;
         try {
             File userHostsFile = new File(user.getAddressbookLocation());
             Properties knownHosts = getKnownHosts(user, true);
             if (knownHosts.containsKey(name)) return "Name is already in use";
         
-            out = new FileWriter(userHostsFile, true);
+            out = new FileOutputStream(userHostsFile, true);
-            out.write(name + "=" + location + '\n');
+            out.write(DataHelper.getUTF8(name + "=" + location + '\n'));
             return "Address " + name + " written to your hosts file (" + userHostsFile.getName() + ")";
         } catch (IOException ioe) {
             return "Error writing out host entry: " + ioe.getMessage();

apps/syndie/java/src/net/i2p/syndie/CLI.java

@@ -116,7 +116,7 @@ public class CLI {
             boolean showImages = "true".equalsIgnoreCase(args[6]);
             try {
                 File f = File.createTempFile("syndie", ".html");
-                Writer out = new FileWriter(f);
+                Writer out = new OutputStreamWriter(new FileOutputStream(f), "UTF-8");
                 renderer.render(null, mgr.getArchive(), entry, out, summaryOnly, showImages);
                 out.flush();
                 out.close();

apps/syndie/java/src/net/i2p/syndie/CachedEntry.java

@@ -21,7 +21,7 @@ class CachedEntry extends EntryContainer {
     private Entry _entry;
     private Attachment _attachments[];
     
-    public CachedEntry(File entryDir) {
+    public CachedEntry(File entryDir) throws IOException {
         _entryDir = entryDir;
         importMeta();
         _entry = new CachedEntryDetails();
@@ -107,7 +107,7 @@ class CachedEntry extends EntryContainer {
         Properties rv = new Properties();
         BufferedReader in = null;
         try {
-            in = new BufferedReader(new FileReader(propsFile));
+            in = new BufferedReader(new InputStreamReader(new FileInputStream(propsFile), "UTF-8"));
             String line = null;
             while ( (line = in.readLine()) != null) {
                 int split = line.indexOf('=');
@@ -152,7 +152,7 @@ class CachedEntry extends EntryContainer {
                     in = new FileInputStream(f);
                     int read = DataHelper.read(in, buf);
                     if (read != buf.length) throw new IOException("read: " + read + " file size: " + buf.length + " for " + f.getPath());
-                    _text = new String(buf);
+                    _text = DataHelper.getUTF8(buf);
                 } catch (IOException ioe) {
                     ioe.printStackTrace();
                 } finally {

apps/syndie/java/src/net/i2p/syndie/EntryExtractor.java

@@ -65,36 +65,36 @@ public class EntryExtractor {
         }
     }
     private void extractHeaders(EntryContainer entry, File entryDir) throws IOException {
-        FileWriter out = null;
+        FileOutputStream out = null;
         try {
-            out = new FileWriter(new File(entryDir, HEADERS));
+            out = new FileOutputStream(new File(entryDir, HEADERS));
             Map headers = entry.getHeaders();
             for (Iterator iter = headers.keySet().iterator(); iter.hasNext(); ) {
                 String k = (String)iter.next();
                 String v = (String)headers.get(k);
-                out.write(k.trim() + '=' + v.trim() + '\n');
+                out.write(DataHelper.getUTF8(k.trim() + '=' + v.trim() + '\n'));
             }
         } finally {
             out.close();
         }
     }
     private void extractMeta(EntryContainer entry, File entryDir) throws IOException {
-        FileWriter out = null;
+        FileOutputStream out = null;
         try {
-            out = new FileWriter(new File(entryDir, META));
+            out = new FileOutputStream(new File(entryDir, META));
-            out.write("format=" + entry.getFormat() + '\n');
+            out.write(DataHelper.getUTF8("format=" + entry.getFormat() + '\n'));
-            out.write("size=" + entry.getCompleteSize() + '\n');
+            out.write(DataHelper.getUTF8("size=" + entry.getCompleteSize() + '\n'));
-            out.write("blog=" + entry.getURI().getKeyHash().toBase64() + '\n');
+            out.write(DataHelper.getUTF8("blog=" + entry.getURI().getKeyHash().toBase64() + '\n'));
-            out.write("entry=" + entry.getURI().getEntryId() + '\n');
+            out.write(DataHelper.getUTF8("entry=" + entry.getURI().getEntryId() + '\n'));
         } finally {
             out.close();
         }
     }
     private void extractEntry(EntryContainer entry, File entryDir) throws IOException {
-        FileWriter out = null;
+        FileOutputStream out = null;
         try {
-            out = new FileWriter(new File(entryDir, ENTRY));
+            out = new FileOutputStream(new File(entryDir, ENTRY));
-            out.write(entry.getEntry().getText());
+            out.write(DataHelper.getUTF8(entry.getEntry().getText()));
         } finally {
             out.close();
         }
@@ -115,16 +115,16 @@ public class EntryExtractor {
         }
     }
     private void extractAttachmentMetadata(int num, Attachment attachment, File entryDir) throws IOException {
-        FileWriter out = null;
+        FileOutputStream out = null;
         try {
-            out = new FileWriter(new File(entryDir, ATTACHMENT_PREFIX + num + ATTACHMENT_META_SUFFIX));
+            out = new FileOutputStream(new File(entryDir, ATTACHMENT_PREFIX + num + ATTACHMENT_META_SUFFIX));
             Map meta = attachment.getMeta();
             for (Iterator iter = meta.keySet().iterator(); iter.hasNext(); ) {
                 String k = (String)iter.next();
                 String v = (String)meta.get(k);
-                out.write(k + '=' + v + '\n');
+                out.write(DataHelper.getUTF8(k + '=' + v + '\n'));
             }
-            out.write(ATTACHMENT_DATA_SIZE + '=' + attachment.getDataLength());
+            out.write(DataHelper.getUTF8(ATTACHMENT_DATA_SIZE + '=' + attachment.getDataLength()));
         } finally {
             out.close();
         }

apps/syndie/java/src/net/i2p/syndie/User.java

@@ -1,5 +1,6 @@
 package net.i2p.syndie;
 
+import java.io.UnsupportedEncodingException;
 import java.util.*;
 import net.i2p.I2PAppContext;
 import net.i2p.data.*;
@@ -98,7 +99,7 @@ public class User {
     
     public String login(String login, String pass, Properties props) {
         String expectedPass = props.getProperty("password");
-        String hpass = Base64.encode(_context.sha().calculateHash(pass.getBytes()).getData());
+        String hpass = Base64.encode(_context.sha().calculateHash(DataHelper.getUTF8(pass)).getData());
         if (!hpass.equals(expectedPass)) {
             _authenticated = false;
             return "Incorrect password";
@@ -195,12 +196,6 @@ public class User {
         buf.append("showexpanded=" + getShowExpanded() + "\n");
         buf.append("defaultselector=" + getDefaultSelector() + "\n");
         buf.append("allowaccessremote=" + _allowAccessRemote + "\n");
-        buf.append("eepproxyhost="+_eepProxyHost+"\n");
-        buf.append("eepproxyport="+_eepProxyPort+"\n");
-        buf.append("webproxyhost="+_webProxyHost+"\n");
-        buf.append("webproxyport="+_webProxyPort+"\n");
-        buf.append("torproxyhost="+_torProxyHost+"\n");
-        buf.append("torproxyport="+_torProxyPort+"\n");
         
         buf.append("groups=");
         Map groups = getBlogGroups();

apps/syndie/java/src/net/i2p/syndie/data/ArchiveIndex.java

@@ -190,7 +190,7 @@ public class ArchiveIndex {
         _newestBlogs = new ArrayList();
         _newestEntries = new ArrayList();
         _headers = new Properties();
-        BufferedReader in = new BufferedReader(new InputStreamReader(index));
+        BufferedReader in = new BufferedReader(new InputStreamReader(index, "UTF-8"));
         String line = null;
         line = in.readLine();
         if (line == null)
@@ -240,8 +240,8 @@ public class ArchiveIndex {
                 _newestBlogs = parseNewestBlogs(val);
             else if (key.equals("NewestEntries"))
                 _newestEntries = parseNewestEntries(val);
-            else
+            //else
-                System.err.println("Key: " + key + " val: " + val);
+            //    System.err.println("Key: " + key + " val: " + val);
         }
     }
     
@@ -265,6 +265,19 @@ public class ArchiveIndex {
             if (tag != null) {
                 if (!tag.equals(summary.tag)) {
                     System.out.println("Tag [" + summary.tag + "] does not match the requested [" + tag + "] in " + summary.blog.toBase64());
+                    if (false) {
+                        StringBuffer b = new StringBuffer(tag.length()*2);
+                        for (int j = 0; j < tag.length(); j++) {
+                            b.append((int)tag.charAt(j));
+                            b.append('.');
+                            if (summary.tag.length() > j+1)
+                                b.append((int)summary.tag.charAt(j));
+                            else
+                                b.append('_');
+                            b.append(' ');
+                        }
+                        System.out.println("tag.summary: " + b.toString());
+                    }
                     continue;
                 }
             }
@@ -273,7 +286,7 @@ public class ArchiveIndex {
                 EntrySummary entry = (EntrySummary)summary.entries.get(j);
                 String k = (Long.MAX_VALUE-entry.entry.getEntryId()) + "-" + entry.entry.getKeyHash().toBase64();
                 ordered.put(k, entry.entry);
-                System.err.println("Including match: " + k);
+                //System.err.println("Including match: " + k);
             }
         }
         for (Iterator iter = ordered.values().iterator(); iter.hasNext(); ) {
@@ -313,8 +326,10 @@ public class ArchiveIndex {
         if (tok.countTokens() < 4)
             return;
         tok.nextToken();
-        Hash keyHash = new Hash(Base64.decode(tok.nextToken()));
+        String keyStr = tok.nextToken();
-        long when = getIndexDate(tok.nextToken());
+        Hash keyHash = new Hash(Base64.decode(keyStr));
+        String whenStr = tok.nextToken();
+        long when = getIndexDate(whenStr);
         String tag = tok.nextToken();
         BlogSummary summary = new BlogSummary();
         summary.blog = keyHash;

apps/syndie/java/src/net/i2p/syndie/data/Attachment.java

@@ -2,6 +2,7 @@ package net.i2p.syndie.data;
 
 import java.io.*;
 import java.util.*;
+import net.i2p.data.DataHelper;
 
 /**
  *
@@ -85,7 +86,7 @@ public class Attachment {
         for (int i = 0; i < _keys.size(); i++) {
             meta.append(_keys.get(i)).append(':').append(_values.get(i)).append('\n');
         }
-        _rawMetadata = meta.toString().getBytes();
+        _rawMetadata = DataHelper.getUTF8(meta);
     }
     
     private void parseMeta() {
@@ -96,10 +97,10 @@ public class Attachment {
         int valBegin = -1;
         for (int i = 0; i < _rawMetadata.length; i++) {
             if (_rawMetadata[i] == ':') {
-                key = new String(_rawMetadata, keyBegin, i - keyBegin);
+                key = DataHelper.getUTF8(_rawMetadata, keyBegin, i - keyBegin);
                 valBegin = i + 1;
             } else if (_rawMetadata[i] == '\n') {
-                val = new String(_rawMetadata, valBegin, i - valBegin);
+                val = DataHelper.getUTF8(_rawMetadata, valBegin, i - valBegin);
                 _keys.add(key);
                 _values.add(val);
                 keyBegin = i + 1;

apps/syndie/java/src/net/i2p/syndie/data/BlogInfo.java

@@ -57,11 +57,12 @@ public class BlogInfo {
     public static final String EDITION = "Edition";
     
     public void load(InputStream in) throws IOException {
-        BufferedReader reader = new BufferedReader(new InputStreamReader(in));
+        BufferedReader reader = new BufferedReader(new InputStreamReader(in, "UTF-8"));
         List names = new ArrayList();
         List vals = new ArrayList();
         String line = null;
         while ( (line = reader.readLine()) != null) {
+            System.err.println("Read info line [" + line + "]");
             line = line.trim();
             int len = line.length();
             int split = line.indexOf(':');
@@ -83,6 +84,7 @@ public class BlogInfo {
         for (int i = 0; i < _optionNames.length; i++) {
             _optionNames[i] = (String)names.get(i);
             _optionValues[i] = (String)vals.get(i);
+            System.out.println("Loaded info: [" + _optionNames[i] + "] = [" + _optionValues[i] + "]");
         }
         
         String keyStr = getProperty(OWNER_KEY);
@@ -110,13 +112,21 @@ public class BlogInfo {
                 buf.append(_optionNames[i]).append(':').append(_optionValues[i]).append('\n');
         }
         String s = buf.toString();
-        out.write(s.getBytes());
+        out.write(s.getBytes("UTF-8"));
     }
     
     public String getProperty(String name) {
         for (int i = 0; i < _optionNames.length; i++) {
-            if (_optionNames[i].equals(name))
+            if (_optionNames[i].equals(name)) {
-                return _optionValues[i];
+                String val = _optionValues[i];
+                System.out.println("getProperty[" + name + "] = [" + val + "] [sz=" + val.length() +"]");
+                for (int j = 0; j < val.length(); j++) {
+                    char c = (char)val.charAt(j);
+                    if (c != (c & 0x7F))
+                        System.out.println("char " + j + ": " + (int)c);
+                }
+                return val;
+            }
         }
         return null;
     }
@@ -214,31 +224,28 @@ public class BlogInfo {
         }
         return buf.toString();
     }
+
+    private static final String TEST_STRING = "\u20AC\u00DF\u6771\u10400\u00F6";
     
     public static void main(String args[]) {
         I2PAppContext ctx = I2PAppContext.getGlobalContext();
-        /*
+        if (true) {
         try {
             Object keys[] = ctx.keyGenerator().generateSigningKeypair();
             SigningPublicKey pub = (SigningPublicKey)keys[0];
             SigningPrivateKey priv = (SigningPrivateKey)keys[1];
 
             Properties opts = new Properties();
-            opts.setProperty("Name", "n4m3");
+            opts.setProperty("Name", TEST_STRING);
-            opts.setProperty("Description", "foo");
+            opts.setProperty("Description", TEST_STRING);
             opts.setProperty("Edition", "0");
-            opts.setProperty("ContactURL", "u@h.org");
+            opts.setProperty("ContactURL", TEST_STRING);
 
+            String nameOrig = opts.getProperty("Name");
             BlogInfo info = new BlogInfo(pub, null, opts);
-            System.err.println("\n");
-            System.err.println("\n");
             info.sign(ctx, priv);
-            System.err.println("\n");
             boolean ok = info.verify(ctx);
-            System.err.println("\n");
             System.err.println("sign&verify: " + ok);
-            System.err.println("\n");
-            System.err.println("\n");
             
             FileOutputStream o = new FileOutputStream("bloginfo-test.dat");
             info.write(o, true);
@@ -252,8 +259,12 @@ public class BlogInfo {
             System.err.println("write to disk, verify read: " + ok);
             System.err.println("Data: " + Base64.encode(buf, 0, sz));
             System.err.println("Str : " + new String(buf, 0, sz));
+            
+            System.err.println("Name ok? " + read.getProperty("Name").equals(TEST_STRING));
+            System.err.println("Desc ok? " + read.getProperty("Description").equals(TEST_STRING));
+            System.err.println("Name ok? " + read.getProperty("ContactURL").equals(TEST_STRING));
         } catch (Exception e) { e.printStackTrace(); }
-        */
+        } else {
         try {
             FileInputStream in = new FileInputStream(args[0]);
             BlogInfo info = new BlogInfo();
@@ -261,5 +272,6 @@ public class BlogInfo {
             boolean ok = info.verify(I2PAppContext.getGlobalContext());
             System.out.println("OK? " + ok + " :" + info);
         } catch (Exception e) { e.printStackTrace(); }
+        }
     }
 }

apps/syndie/java/src/net/i2p/syndie/data/EncodingTestGenerator.java

@@ -0,0 +1,86 @@
+package net.i2p.syndie.data;
+
+import java.io.*;
+import java.util.*;
+import net.i2p.data.*;
+import net.i2p.I2PAppContext;
+
+/**
+ * Create a new blog metadata & set of entries using some crazy UTF8 encoded chars,
+ * then make sure they're always valid.  These blogs & entries can then be fed into
+ * jetty/syndie/etc to see how and where they are getting b0rked.
+ */
+public class EncodingTestGenerator {
+    public EncodingTestGenerator() {}
+    public static final String TEST_STRING = "\u20AC\u00DF\u6771\u10400\u00F6";
+    
+    public static void main(String args[]) {
+        I2PAppContext ctx = I2PAppContext.getGlobalContext();
+        try {
+            Object keys[] = ctx.keyGenerator().generateSigningKeypair();
+            SigningPublicKey pub = (SigningPublicKey)keys[0];
+            SigningPrivateKey priv = (SigningPrivateKey)keys[1];
+
+            Properties opts = new Properties();
+            opts.setProperty("Name", TEST_STRING);
+            opts.setProperty("Description", TEST_STRING);
+            opts.setProperty("Edition", "0");
+            opts.setProperty("ContactURL", TEST_STRING);
+
+            String nameOrig = opts.getProperty("Name");
+            BlogInfo info = new BlogInfo(pub, null, opts);
+            info.sign(ctx, priv);
+            boolean ok = info.verify(ctx);
+            System.err.println("sign&verify: " + ok);
+            
+            FileOutputStream o = new FileOutputStream("encodedMeta.dat");
+            info.write(o, true);
+            o.close();
+            FileInputStream i = new FileInputStream("encodedMeta.dat");
+            byte buf[] = new byte[4096];
+            int sz = DataHelper.read(i, buf);
+            BlogInfo read = new BlogInfo();
+            read.load(new ByteArrayInputStream(buf, 0, sz));
+            ok = read.verify(ctx);
+            System.err.println("write to disk, verify read: " + ok);
+            System.err.println("Name ok? " + read.getProperty("Name").equals(TEST_STRING));
+            System.err.println("Desc ok? " + read.getProperty("Description").equals(TEST_STRING));
+            System.err.println("Name ok? " + read.getProperty("ContactURL").equals(TEST_STRING));
+            
+            // ok now lets create some entries
+            BlogURI uri = new BlogURI(read.getKey().calculateHash(), 0);
+            String tags[] = new String[4];
+            for (int j = 0; j < tags.length; j++)
+                tags[j] = TEST_STRING + "_" + j;
+            StringBuffer smlOrig = new StringBuffer(512);
+            smlOrig.append("Subject: ").append(TEST_STRING).append("\n\n");
+            smlOrig.append("Hi with ").append(TEST_STRING);
+            EntryContainer container = new EntryContainer(uri, tags, DataHelper.getUTF8(smlOrig));
+            container.seal(ctx, priv, null);
+            ok = container.verifySignature(ctx, read);
+            System.err.println("Sealed and verified entry: " + ok);
+            FileOutputStream fos = new FileOutputStream("encodedEntry.dat");
+            container.write(fos, true);
+            fos.close();
+            System.out.println("Written to " + new File("encodedEntry.dat").getAbsolutePath());
+            
+            FileInputStream fis = new FileInputStream("encodedEntry.dat");
+            EntryContainer read2 = new EntryContainer();
+            read2.load(fis);
+            ok = read2.verifySignature(ctx, read);
+            System.out.println("Read ok? " + ok);
+            
+            read2.parseRawData(ctx);
+            String tagsRead[] = read2.getTags();
+            for (int j = 0; j < tagsRead.length; j++) {
+                if (!tags[j].equals(tagsRead[j]))
+                    System.err.println("Tag error [" + j + "]: read = [" + tagsRead[j] + "] want [" + tags[j] + "]");
+                else
+                    System.err.println("Tag ok [" + j + "]");
+            }
+            String readText = read2.getEntry().getText();
+            ok = readText.equals(smlOrig.toString());
+            System.err.println("SML text ok? " + ok);
+        } catch (Exception e) { e.printStackTrace(); }
+    }
+}

apps/syndie/java/src/net/i2p/syndie/data/EntryContainer.java

@@ -58,7 +58,7 @@ public class EntryContainer {
     public EntryContainer(BlogURI uri, String tags[], byte smlData[]) {
         this();
         _entryURI = uri;
-        _entryData = new Entry(new String(smlData));
+        _entryData = new Entry(DataHelper.getUTF8(smlData));
         setHeader(HEADER_BLOGKEY, Base64.encode(uri.getKeyHash().getData()));
         StringBuffer buf = new StringBuffer();
         for (int i = 0; tags != null && i < tags.length; i++)
@@ -71,9 +71,34 @@ public class EntryContainer {
     
     public int getFormat() { return _format; }
     
+    private String readLine(InputStream in) throws IOException {
+        ByteArrayOutputStream baos = new ByteArrayOutputStream(512);
+        int i = 0;
+        while (true) {
+            int c = in.read();
+            if ( (c == (int)'\n') || (c == (int)'\r') ) {
+                break;
+            } else if (c == -1) {
+                if (i == 0)
+                    return null;
+                else
+                    break;
+            } else {
+                baos.write(c);
+            }
+            i++;
+        }
+        
+        return DataHelper.getUTF8(baos.toByteArray());
+        //BufferedReader r = new BufferedReader(new InputStreamReader(in, "UTF-8"), 1);
+        //String line = r.readLine();
+        //return line;
+    }
+    
     public void load(InputStream source) throws IOException {
-        String line = DataHelper.readLine(source);
+        String line = readLine(source);
         if (line == null) throw new IOException("No format line in the entry");
+        //System.err.println("read container format line [" + line + "]");
         String fmt = line.trim();
         if (FORMAT_ZIP_UNENCRYPTED_STR.equals(fmt)) {
             _format = FORMAT_ZIP_UNENCRYPTED;
@@ -83,7 +108,8 @@ public class EntryContainer {
             throw new IOException("Unsupported entry format: " + fmt);
         }
         
-        while ( (line = DataHelper.readLine(source)) != null) {
+        while ( (line = readLine(source)) != null) {
+            //System.err.println("read container header line [" + line + "]");
             line = line.trim();
             int len = line.length();
             if (len <= 0)
@@ -99,7 +125,8 @@ public class EntryContainer {
         
         parseHeaders();
         
-        String sigStr = DataHelper.readLine(source);
+        String sigStr = readLine(source); 
+        //System.err.println("read container signature line [" + line + "]");
         if ( (sigStr == null) || (sigStr.indexOf("Signature:") == -1) )
             throw new IOException("No signature line");
         sigStr = sigStr.substring("Signature:".length()+1).trim();
@@ -107,7 +134,8 @@ public class EntryContainer {
         _signature = new Signature(Base64.decode(sigStr));
         //System.out.println("Sig: " + _signature.toBase64());
         
-        line = DataHelper.readLine(source);
+        line = readLine(source); 
+        //System.err.println("read container size line [" + line + "]");
         if (line == null)
             throw new IOException("No size line");
         line = line.trim();
@@ -165,7 +193,7 @@ public class EntryContainer {
         ByteArrayOutputStream baos = new ByteArrayOutputStream();
         ZipOutputStream out = new ZipOutputStream(baos);
         ZipEntry ze = new ZipEntry(ZIP_ENTRY);
-        byte data[] = _entryData.getText().getBytes();
+        byte data[] = DataHelper.getUTF8(_entryData.getText());
         ze.setTime(0);
         out.putNextEntry(ze);
         out.write(data);
@@ -222,7 +250,7 @@ public class EntryContainer {
             
             String name = entry.getName();
             if (ZIP_ENTRY.equals(name)) {
-                _entryData = new Entry(new String(entryData));
+                _entryData = new Entry(DataHelper.getUTF8(entryData));
             } else if (name.startsWith(ZIP_ATTACHMENT_PREFIX)) {
                 attachments.put(name, (Object)entryData);
             } else if (name.startsWith(ZIP_ATTACHMENT_META_PREFIX)) {
@@ -311,14 +339,19 @@ public class EntryContainer {
         String keyHash = getHeader(HEADER_BLOGKEY);
         String idVal = getHeader(HEADER_ENTRYID);
         
-        if (keyHash == null)
+        if (keyHash == null) {
+            System.err.println("Headers: " + _rawKeys);
+            System.err.println("Values : " + _rawValues);
             throw new IOException("Missing " + HEADER_BLOGKEY + " header");
+        }
         
         long entryId = -1;
         if ( (idVal != null) && (idVal.length() > 0) ) {
             try {
                 entryId = Long.parseLong(idVal.trim());
             } catch (NumberFormatException nfe) {
+                System.err.println("Headers: " + _rawKeys);
+                System.err.println("Values : " + _rawValues);
                 throw new IOException("Invalid format of entryId (" + idVal + ")");
             }
         }
@@ -385,7 +418,7 @@ public class EntryContainer {
         String str = buf.toString();
         
         //System.out.println("Writing raw: \n[" + str + "] / " + I2PAppContext.getGlobalContext().sha().calculateHash(str.getBytes()) + ", raw data: " + I2PAppContext.getGlobalContext().sha().calculateHash(_rawData).toBase64() + "\n");
-        out.write(str.getBytes());
+        out.write(DataHelper.getUTF8(str));
         out.write(_rawData);
     }
     

apps/syndie/java/src/net/i2p/syndie/data/LocalArchiveIndex.java

@@ -75,7 +75,7 @@ public class LocalArchiveIndex extends ArchiveIndex {
             _replies.put(parent, replies);
         }
         replies.add(reply);
-        System.err.println("Adding reply to " + parent + " from child " + reply + " (# replies: " + replies.size() + ")");
+        //System.err.println("Adding reply to " + parent + " from child " + reply + " (# replies: " + replies.size() + ")");
     }
 
     private static class BlogURIComparator implements Comparator {

apps/syndie/java/src/net/i2p/syndie/sml/HTMLRenderer.java

@@ -44,7 +44,7 @@ public class HTMLRenderer extends EventReceiverImpl {
             return;
         }
         HTMLRenderer renderer = new HTMLRenderer();
-        FileWriter out = null;
+        Writer out = null;
         try {
             ByteArrayOutputStream baos = new ByteArrayOutputStream(1024*512);
             FileInputStream in = new FileInputStream(args[0]);
@@ -52,8 +52,8 @@ public class HTMLRenderer extends EventReceiverImpl {
             int read = 0;
             while ( (read = in.read(buf)) != -1)
                 baos.write(buf, 0, read);
-            out = new FileWriter(args[1]);
+            out = new OutputStreamWriter(new FileOutputStream(args[1]), "UTF-8");
-            renderer.render(new User(), BlogManager.instance().getArchive(), null, new String(baos.toByteArray()), out, false, true);   
+            renderer.render(new User(), BlogManager.instance().getArchive(), null, DataHelper.getUTF8(baos.toByteArray()), out, false, true);   
         } catch (IOException ioe) {
             ioe.printStackTrace();
         } finally {
@@ -595,7 +595,7 @@ public class HTMLRenderer extends EventReceiverImpl {
     }
     
     public void receiveHeader(String header, String value) { 
-        System.err.println("Receive header [" + header + "] = [" + value + "]");
+        //System.err.println("Receive header [" + header + "] = [" + value + "]");
         _headers.put(header, value); 
     }
     
@@ -652,7 +652,7 @@ public class HTMLRenderer extends EventReceiverImpl {
             for (int i = 0; tags != null && i < tags.length; i++) {
                 _preBodyBuffer.append("<option value=\"blogtag://");
                 _preBodyBuffer.append(_entry.getURI().getKeyHash().toBase64());
-                _preBodyBuffer.append('/').append(Base64.encode(tags[i])).append("\">");
+                _preBodyBuffer.append('/').append(Base64.encode(DataHelper.getUTF8(tags[i]))).append("\">");
                 _preBodyBuffer.append(sanitizeString(tags[i]));
                 _preBodyBuffer.append("</option>\n");
                 /*
@@ -726,7 +726,7 @@ public class HTMLRenderer extends EventReceiverImpl {
         return str;
     }
 
-    public static final String sanitizeURL(String str) { return Base64.encode(str); }
+    public static final String sanitizeURL(String str) { return Base64.encode(DataHelper.getUTF8(str)); }
     public static final String sanitizeTagParam(String str) {
         str = str.replace('&', '_'); // this should be &amp;
         if (str.indexOf('\"') < 0)
@@ -801,11 +801,11 @@ public class HTMLRenderer extends EventReceiverImpl {
         if (blog != null)
             buf.append(ArchiveViewerBean.PARAM_BLOG).append('=').append(Base64.encode(blog.getData())).append('&');
         if (tag != null)
-            buf.append(ArchiveViewerBean.PARAM_TAG).append('=').append(Base64.encode(tag)).append('&');
+            buf.append(ArchiveViewerBean.PARAM_TAG).append('=').append(Base64.encode(DataHelper.getUTF8(tag))).append('&');
         if (entryId >= 0)
             buf.append(ArchiveViewerBean.PARAM_ENTRY).append('=').append(entryId).append('&');
         if (group != null)
-            buf.append(ArchiveViewerBean.PARAM_GROUP).append('=').append(Base64.encode(group)).append('&');
+            buf.append(ArchiveViewerBean.PARAM_GROUP).append('=').append(Base64.encode(DataHelper.getUTF8(group))).append('&');
         if ( (pageNum >= 0) && (numPerPage > 0) ) {
             buf.append(ArchiveViewerBean.PARAM_PAGE_NUMBER).append('=').append(pageNum).append('&');
             buf.append(ArchiveViewerBean.PARAM_NUM_PER_PAGE).append('=').append(numPerPage).append('&');

apps/syndie/java/src/net/i2p/syndie/web/ArchiveServlet.java

@@ -39,7 +39,7 @@ public class ArchiveServlet extends HttpServlet {
     }
     
     private String getBlog(String path) {
-        System.err.println("Blog: [" + path + "]");
+        //System.err.println("Blog: [" + path + "]");
         int start = 0;
         int end = -1;
         int len = path.length();
@@ -57,7 +57,7 @@ public class ArchiveServlet extends HttpServlet {
         }
         if (end < 0) end = len;
         String rv = path.substring(start, end);
-        System.err.println("Blog: [" + path + "] rv: [" + rv + "]");
+        //System.err.println("Blog: [" + path + "] rv: [" + rv + "]");
         return rv;
     }
     
@@ -66,7 +66,7 @@ public class ArchiveServlet extends HttpServlet {
         if (start < 0) return -1;
         if (!(path.endsWith(".snd"))) return -1;
         String rv = path.substring(start+1, path.length()-".snd".length());
-        System.err.println("Entry: [" + path + "] rv: [" + rv + "]");
+        //System.err.println("Entry: [" + path + "] rv: [" + rv + "]");
         try {
             return Long.parseLong(rv);
         } catch (NumberFormatException nfe) {
@@ -75,24 +75,26 @@ public class ArchiveServlet extends HttpServlet {
     }
     
     private void renderRootIndex(HttpServletResponse resp) throws ServletException, IOException {
-        resp.setContentType("text/html");
+        resp.setContentType("text/html;charset=utf-8");
+        //resp.setCharacterEncoding("UTF-8");
         OutputStream out = resp.getOutputStream();
-        out.write("<a href=\"archive.txt\">archive.txt</a><br />\n".getBytes());
+        out.write(DataHelper.getUTF8("<a href=\"archive.txt\">archive.txt</a><br />\n"));
         ArchiveIndex index = BlogManager.instance().getArchive().getIndex();
         Set blogs = index.getUniqueBlogs();
         for (Iterator iter = blogs.iterator(); iter.hasNext(); ) {
             Hash blog = (Hash)iter.next();
             String s = blog.toBase64();
-            out.write(("<a href=\"" + s + "/\">" + s + "</a><br />\n").getBytes());
+            out.write(DataHelper.getUTF8("<a href=\"" + s + "/\">" + s + "</a><br />\n"));
         }
         out.close();
     }
     
     private void renderSummary(HttpServletResponse resp) throws ServletException, IOException {
-        resp.setContentType("text/plain");
+        resp.setContentType("text/plain;charset=utf-8");
+        //resp.setCharacterEncoding("UTF-8");
         OutputStream out = resp.getOutputStream();
         ArchiveIndex index = BlogManager.instance().getArchive().getIndex();
-        out.write(index.toString().getBytes());
+        out.write(DataHelper.getUTF8(index.toString()));
         out.close();
     }
     
@@ -127,15 +129,16 @@ public class ArchiveServlet extends HttpServlet {
             resp.sendError(404, "Blog does not exist");
             return;
         }
-        resp.setContentType("text/html");
+        resp.setContentType("text/html;charset=utf-8");
+        //resp.setCharacterEncoding("UTF-8");
         OutputStream out = resp.getOutputStream();
-        out.write("<a href=\"..\">..</a><br />\n".getBytes());
+        out.write(DataHelper.getUTF8("<a href=\"..\">..</a><br />\n"));
-        out.write(("<a href=\"" + Archive.METADATA_FILE + "\">" + Archive.METADATA_FILE + "</a><br />\n").getBytes());
+        out.write(DataHelper.getUTF8("<a href=\"" + Archive.METADATA_FILE + "\">" + Archive.METADATA_FILE + "</a><br />\n"));
         List entries = new ArrayList(64);
         BlogManager.instance().getArchive().getIndex().selectMatchesOrderByEntryId(entries, h, null);
         for (int i = 0; i < entries.size(); i++) {
             BlogURI entry = (BlogURI)entries.get(i);
-            out.write(("<a href=\"" + entry.getEntryId() + ".snd\">" + entry.getEntryId() + ".snd</a><br />\n").getBytes());
+            out.write(DataHelper.getUTF8("<a href=\"" + entry.getEntryId() + ".snd\">" + entry.getEntryId() + ".snd</a><br />\n"));
         }
         out.close();
     }

apps/syndie/java/src/net/i2p/syndie/web/ArchiveViewerBean.java

@@ -108,7 +108,7 @@ public class ArchiveViewerBean {
         if (groups != null) {
             for (Iterator iter = groups.keySet().iterator(); iter.hasNext(); ) {
                 String name = (String)iter.next();
-                out.write("<option value=\"group://" + Base64.encode(name.getBytes()) + "\">" +
+                out.write("<option value=\"group://" + Base64.encode(DataHelper.getUTF8(name)) + "\">" +
                           "Group: " + HTMLRenderer.sanitizeString(name) + "</option>\n");
             }
         }
@@ -152,12 +152,38 @@ public class ArchiveViewerBean {
             List tags = index.getBlogTags(cur);
             for (int j = 0; j < tags.size(); j++) {
                 String tag = (String)tags.get(j);
+                if (false) {
+                    StringBuffer b = new StringBuffer(tag.length()*2);
+                    for (int k = 0; k < tag.length(); k++) {
+                        b.append((int)tag.charAt(k));
+                        b.append(' ');
+                    }
+                    System.out.println("tag in select: " + tag + ": " + b.toString());
+                }
+                
                 if (!allTags.contains(tag))
                     allTags.add(tag);
                 out.write("<option value=\"blogtag://");
                 out.write(blog);
                 out.write("/");
-                out.write(Base64.encode(tag));
+                byte utf8tag[] = DataHelper.getUTF8(tag);
+                String encoded = Base64.encode(utf8tag);
+                if (false) {
+                    byte utf8dec[] = Base64.decode(encoded);
+                    String travel = DataHelper.getUTF8(utf8dec);
+                    StringBuffer b = new StringBuffer();
+                    for (int k = 0; k < travel.length(); k++) {
+                        b.append((int)travel.charAt(k));
+                        b.append(' ');
+                    }
+                    b.append(" encoded into: ");
+                    for (int k = 0; k < encoded.length(); k++) {
+                        b.append((int)encoded.charAt(k));
+                        b.append(' ');
+                    }
+                    System.out.println("UTF8(unbase64(base64(UTF8(tag)))) == tag: " + b.toString());
+                }
+                out.write(encoded);
                 out.write("\">");
                 out.write(name);
                 out.write("- posts with the tag &quot;");
@@ -168,7 +194,7 @@ public class ArchiveViewerBean {
         for (int i = 0; i < allTags.size(); i++) {
             String tag = (String)allTags.get(i);
             out.write("<option value=\"tag://");
-            out.write(Base64.encode(tag));
+            out.write(Base64.encode(DataHelper.getUTF8(tag)));
             out.write("\">Posts in any blog with the tag &quot;");
             out.write(tag);
             out.write("&quot;</option>\n");
@@ -199,7 +225,7 @@ public class ArchiveViewerBean {
         Hash blog = null;
         if (blogStr != null) blog = new Hash(Base64.decode(blogStr));
         String tag = getString(parameters, PARAM_TAG);
-        if (tag != null) tag = new String(Base64.decode(tag));
+        if (tag != null) tag = DataHelper.getUTF8(Base64.decode(tag));
         long entryId = -1;
         if (blogStr != null) {
             String entryIdStr = getString(parameters, PARAM_ENTRY);
@@ -208,7 +234,7 @@ public class ArchiveViewerBean {
             } catch (NumberFormatException nfe) {}
         }
         String group = getString(parameters, PARAM_GROUP);
-        if (group != null) group = new String(Base64.decode(group));
+        if (group != null) group = DataHelper.getUTF8(Base64.decode(group));
         
         String sel = getString(parameters, PARAM_SELECTOR);
         if ( (sel == null) && (blog == null) && (group == null) && (tag == null) )
@@ -256,15 +282,48 @@ public class ArchiveViewerBean {
                     String blogStr = selector.substring(SEL_BLOGTAG.length(), tagStart);
                     blog = new Hash(Base64.decode(blogStr));
                     tag = selector.substring(tagStart+1);
-                    if (tag != null) tag = new String(Base64.decode(tag));
+                    String origTag = tag;
+                    byte rawDecode[] = null;
+                    if (tag != null) {
+                        rawDecode = Base64.decode(tag);
+                        tag = DataHelper.getUTF8(rawDecode);
+                    }
                     System.out.println("Selector [" + selector + "] blogString: [" + blogStr + "] tag: [" + tag + "]");
+                    if (false && tag != null) {
+                        StringBuffer b = new StringBuffer(tag.length()*2);
+                        for (int j = 0; j < tag.length(); j++) {
+                            b.append((int)tag.charAt(j));
+                            if (rawDecode.length > j)
+                                b.append('.').append((int)rawDecode[j]);
+                            b.append(' ');
+                        }
+                        b.append("encoded as ");
+                        for (int j = 0; j < origTag.length(); j++) {
+                            b.append((int)origTag.charAt(j)).append(' ');
+                        }
+                        System.out.println("selected tag: " + b.toString());
+                    }
                 } else if (selector.startsWith(SEL_TAG)) {
                     tag = selector.substring(SEL_TAG.length());
-                    if (tag != null) tag = new String(Base64.decode(tag));
+                    byte rawDecode[] = null;
+                    if (tag != null) {
+                        rawDecode = Base64.decode(tag);
+                        tag = DataHelper.getUTF8(rawDecode);
+                    }
                     System.out.println("Selector [" + selector + "] tag: [" + tag + "]");
+                    if (false && tag != null) {
+                        StringBuffer b = new StringBuffer(tag.length()*2);
+                        for (int j = 0; j < tag.length(); j++) {
+                            b.append((int)tag.charAt(j));
+                            if (rawDecode.length > j)
+                                b.append('.').append((int)rawDecode[j]);
+                            b.append(' ');
+                        }
+                        System.out.println("selected tag: " + b.toString());
+                    }
                 } else if (selector.startsWith(SEL_ENTRY)) {
                     int entryStart = selector.lastIndexOf('/');
-                    String blogStr = selector.substring(SEL_ENTRY.length(), entryStart);
+                    String blogStr = blogStr = selector.substring(SEL_ENTRY.length(), entryStart);
                     String entryStr = selector.substring(entryStart+1);
                     try {
                         entry = Long.parseLong(entryStr);
@@ -272,7 +331,7 @@ public class ArchiveViewerBean {
                         System.out.println("Selector [" + selector + "] blogString: [" + blogStr + "] entry: [" + entry + "]");
                     } catch (NumberFormatException nfe) {}
                 } else if (selector.startsWith(SEL_GROUP)) {
-                    group = new String(Base64.decode(selector.substring(SEL_GROUP.length())));
+                    group = DataHelper.getUTF8(Base64.decode(selector.substring(SEL_GROUP.length())));
                     System.out.println("Selector [" + selector + "] group: [" + group + "]");
                 }
             }
@@ -510,7 +569,7 @@ public class ArchiveViewerBean {
     }
     
     private static void renderInvalidAttachment(Map parameters, OutputStream out) throws IOException {
-        out.write("<b>No such entry, or no such attachment</b>".getBytes());
+        out.write(DataHelper.getUTF8("<b>No such entry, or no such attachment</b>"));
     }
     
     public static void renderMetadata(Map parameters, Writer out) throws IOException {

apps/syndie/java/src/net/i2p/syndie/web/RemoteArchiveBean.java

@@ -581,7 +581,7 @@ public class RemoteArchiveBean {
         for (Iterator iter = localBlogs.iterator(); iter.hasNext(); ) {
             Hash blog = (Hash)iter.next();
             if (remoteBlogs.contains(blog)) {
-                System.err.println("Remote index has " + blog.toBase64());
+                //System.err.println("Remote index has " + blog.toBase64());
                 continue;
             }
             

apps/syndie/jsp/_bodyindex.jsp

@@ -1,7 +1,9 @@
-<%@page import="net.i2p.syndie.web.ArchiveViewerBean, net.i2p.syndie.*" %>
+<%@page contentType="text/html; charset=UTF-8" import="net.i2p.syndie.web.ArchiveViewerBean, net.i2p.syndie.*" %>
+<% request.setCharacterEncoding("UTF-8"); %>
 <jsp:useBean scope="session" class="net.i2p.syndie.User" id="user" /><table border="0" width="100%">
 <tr><form action="index.jsp"><td nowrap="true">
 <b>Blogs:</b> <%ArchiveViewerBean.renderBlogSelector(user, request.getParameterMap(), out);%>
 <input type="submit" value="Refresh" />
 <input type="submit" name="action" value="<%=ArchiveViewerBean.SEL_ACTION_SET_AS_DEFAULT%>" />
+<!-- char encoding: [<%=response.getCharacterEncoding()%>] content type [<%=response.getContentType()%>] Locale [<%=response.getLocale()%>] -->
 <%ArchiveViewerBean.renderBlogs(user, request.getParameterMap(), out, "</td></form></tr><tr><td align=\"left\" valign=\"top\">");%></td></tr></table>

apps/syndie/jsp/addaddress.jsp

@@ -1,4 +1,5 @@
-<%@page import="net.i2p.data.Base64, net.i2p.syndie.web.*, net.i2p.syndie.sml.*, net.i2p.syndie.*" %>
+<%@page contentType="text/html; charset=UTF-8" pageEncoding="UTF-8" import="net.i2p.data.*, net.i2p.syndie.web.*, net.i2p.syndie.sml.*, net.i2p.syndie.*" %>
+<% request.setCharacterEncoding("UTF-8"); %>
 <jsp:useBean scope="session" class="net.i2p.syndie.User" id="user" />
 <html>
 <head>
@@ -19,9 +20,9 @@ String name = null;
 String location = null;
 String schema = null;
 try {
-    name = new String(Base64.decode(nameStr));
+    name = DataHelper.getUTF8(Base64.decode(nameStr));
-    location = new String(Base64.decode(locStr));
+    location = DataHelper.getUTF8(Base64.decode(locStr));
-    schema = new String(Base64.decode(schemaStr));
+    schema = DataHelper.getUTF8(Base64.decode(schemaStr));
 } catch (NullPointerException npe) {
     // ignore
 }

apps/syndie/jsp/externallink.jsp

@@ -1,4 +1,5 @@
-<%@page import="net.i2p.data.Base64, net.i2p.syndie.web.*, net.i2p.syndie.sml.*" %>
+<%@page contentType="text/html; charset=UTF-8" pageEncoding="UTF-8" import="net.i2p.data.*, net.i2p.syndie.web.*, net.i2p.syndie.sml.*" %>
+<% request.setCharacterEncoding("UTF-8"); %>
 <html>
 <head>
 <title>SyndieMedia</title>
@@ -15,9 +16,9 @@
 String loc = request.getParameter("location");
 String schema = request.getParameter("schema");
 String desc = request.getParameter("description");
-if (loc != null) loc = HTMLRenderer.sanitizeString(new String(Base64.decode(loc)));
+if (loc != null) loc = HTMLRenderer.sanitizeString(DataHelper.getUTF8(Base64.decode(loc)));
-if (schema != null) schema = HTMLRenderer.sanitizeString(new String(Base64.decode(schema)));
+if (schema != null) schema = HTMLRenderer.sanitizeString(DataHelper.getUTF8(Base64.decode(schema)));
-if (desc != null) desc = HTMLRenderer.sanitizeString(new String(Base64.decode(desc)));
+if (desc != null) desc = HTMLRenderer.sanitizeString(DataHelper.getUTF8(Base64.decode(desc)));
 
 if ( (loc != null) && (schema != null) ) { 
   out.write(loc + " (" + schema + ")"); 

apps/syndie/jsp/import.jsp

@@ -1,4 +1,5 @@
-<%@page import="net.i2p.data.Base64, net.i2p.syndie.web.*, net.i2p.syndie.sml.*, net.i2p.syndie.data.*, net.i2p.syndie.*, org.mortbay.servlet.MultiPartRequest, java.util.*, java.io.*" %>
+<%@page contentType="text/html; charset=UTF-8" pageEncoding="UTF-8" import="net.i2p.data.Base64, net.i2p.syndie.web.*, net.i2p.syndie.sml.*, net.i2p.syndie.data.*, net.i2p.syndie.*, org.mortbay.servlet.MultiPartRequest, java.util.*, java.io.*" %>
+<% request.setCharacterEncoding("UTF-8"); %>
 <jsp:useBean scope="session" class="net.i2p.syndie.data.TransparentArchiveIndex" id="archive" />
 <html>
 <head>

apps/syndie/jsp/index.jsp

@@ -1,4 +1,5 @@
-<%@page contentType="text/html" import="net.i2p.syndie.web.*" %>
+<%@page contentType="text/html; charset=UTF-8" pageEncoding="UTF-8" import="net.i2p.syndie.web.*" %>
+<% request.setCharacterEncoding("UTF-8"); %>
 <html>
 <head>
 <title>SyndieMedia</title>

apps/syndie/jsp/post.jsp

@@ -1,4 +1,5 @@
-<%@page import="net.i2p.data.Base64, net.i2p.syndie.web.*, net.i2p.syndie.sml.*, net.i2p.syndie.data.*, net.i2p.syndie.*, org.mortbay.servlet.MultiPartRequest, java.util.*" %>
+<%@page contentType="text/html; charset=UTF-8" pageEncoding="UTF-8" import="net.i2p.data.Base64, net.i2p.syndie.web.*, net.i2p.syndie.sml.*, net.i2p.syndie.data.*, net.i2p.syndie.*, org.mortbay.servlet.MultiPartRequest, java.util.*" %>
+<% request.setCharacterEncoding("UTF-8"); %>
 <jsp:useBean scope="session" class="net.i2p.syndie.User" id="user" />
 <jsp:useBean scope="session" class="net.i2p.syndie.web.PostBean" id="post" />
 <html>

apps/syndie/jsp/register.jsp

@@ -1,4 +1,5 @@
-<%@page import="net.i2p.data.Base64, net.i2p.syndie.web.*, net.i2p.syndie.sml.*, net.i2p.syndie.*" %>
+<%@page contentType="text/html; charset=UTF-8" pageEncoding="UTF-8" import="net.i2p.data.Base64, net.i2p.syndie.web.*, net.i2p.syndie.sml.*, net.i2p.syndie.*" %>
+<% request.setCharacterEncoding("UTF-8"); %>
 <jsp:useBean scope="session" class="net.i2p.syndie.User" id="user" />
 <html>
 <head>
@@ -12,7 +13,6 @@
     <jsp:include page="_topnav.jsp" />
     <td valign="top" align="left" rowspan="2"><jsp:include page="_rightnav.jsp" /></td></tr>
 <tr><td valign="top" align="left" colspan="3"><%
-
 String regLogin = request.getParameter("login");
 boolean showForm = true;
 if ( (regLogin != null) && ("Register".equals(request.getParameter("Register"))) ) {

apps/syndie/jsp/remote.jsp

@@ -1,4 +1,5 @@
-<%@page contentType="text/html" import="net.i2p.syndie.web.*" %>
+<%@page contentType="text/html; charset=UTF-8" pageEncoding="UTF-8" import="net.i2p.syndie.web.*" %>
+<% request.setCharacterEncoding("UTF-8"); %>
 <jsp:useBean scope="session" class="net.i2p.syndie.web.RemoteArchiveBean" id="remote" />
 <jsp:useBean scope="session" class="net.i2p.syndie.User" id="user" />
 <jsp:useBean scope="session" class="net.i2p.syndie.data.TransparentArchiveIndex" id="archive" />

apps/syndie/jsp/style.jsp

@@ -1,2 +1,3 @@
-<%@page contentType="text/css" %>
+<%@page contentType="text/css; charset=UTF-8" pageEncoding="UTF-8"  %>
+<% request.setCharacterEncoding("UTF-8"); %>
 <%@include file="syndie.css" %>

apps/syndie/jsp/viewattachment.jsp

@@ -1,3 +1,4 @@
+<% request.setCharacterEncoding("UTF-8"); %>
 <%
 java.util.Map params = request.getParameterMap();
 response.setContentType(net.i2p.syndie.web.ArchiveViewerBean.getAttachmentContentType(params));

apps/syndie/jsp/viewmetadata.jsp

@@ -1,4 +1,5 @@
-<%@page contentType="text/html" import="net.i2p.syndie.web.*" %>
+<%@page contentType="text/html; charset=UTF-8" pageEncoding="UTF-8" import="net.i2p.syndie.web.*" %>
+<% request.setCharacterEncoding("UTF-8"); %>
 <html>
 <head>
 <title>SyndieMedia</title>

apps/syndie/jsp/viewtempattachment.jsp

@@ -1,5 +1,6 @@
 <%@page  import="net.i2p.syndie.web.ArchiveViewerBean" %><jsp:useBean 
 scope="session" class="net.i2p.syndie.web.PostBean" id="post" /><%
+request.setCharacterEncoding("UTF-8");
 String id = request.getParameter(ArchiveViewerBean.PARAM_ATTACHMENT);
 if (id != null) {
   try {

core/java/src/net/i2p/data/DataHelper.java

@@ -21,6 +21,7 @@ import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.OutputStream;
 import java.io.PrintWriter;
+import java.io.UnsupportedEncodingException;
 import java.math.BigInteger;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -880,5 +881,35 @@ public class DataHelper {
         ReusableGZIPInputStream.release(in);
         return rv;
     }
+
+    public static byte[] getUTF8(String orig) {
+        if (orig == null) return null;
+        try {
+            return orig.getBytes("UTF-8");
+        } catch (UnsupportedEncodingException uee) {
+            throw new RuntimeException("no utf8!?");
+        }
+    }
+    public static byte[] getUTF8(StringBuffer orig) {
+        if (orig == null) return null;
+        return getUTF8(orig.toString());
+    }
+    public static String getUTF8(byte orig[]) {
+        if (orig == null) return null;
+        try {
+            return new String(orig, "UTF-8");
+        } catch (UnsupportedEncodingException uee) {
+            throw new RuntimeException("no utf8!?");
+        }
+    }
+    public static String getUTF8(byte orig[], int offset, int len) {
+        if (orig == null) return null;
+        try {
+            return new String(orig, offset, len, "UTF-8");
+        } catch (UnsupportedEncodingException uee) {
+            throw new RuntimeException("No utf8!?");
+        }
+    }
+    
     
 }