package net.i2p.router.networkdb.kademlia;
/*
 * free (adj.): unencumbered; not under the control of others
 * Written by jrandom in 2003 and released into the public domain
 * with no warranty of any kind, either expressed or implied.
 * It probably won't make your computer catch on fire, or eat
 * your children, but it might.  Use at your own risk.
 *
 */

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FilenameFilter;
import java.io.Flushable;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.concurrent.ConcurrentHashMap;

import net.i2p.data.Base64;
import net.i2p.data.DatabaseEntry;
import net.i2p.data.DataFormatException;
import net.i2p.data.Hash;
import net.i2p.data.router.RouterInfo;
import net.i2p.router.JobImpl;
import net.i2p.router.Router;
import net.i2p.router.RouterContext;
import net.i2p.util.FileUtil;
import net.i2p.util.I2PThread;
import net.i2p.util.Log;
import net.i2p.util.SecureDirectory;
import net.i2p.util.SecureFileOutputStream;

/**
 * Write out keys to disk when we get them and periodically read ones we don't know
 * about into memory, with newly read routers are also added to the routing table.
 *
 * Public only for access to static methods by startup classes
 *
 */
public class PersistentDataStore extends TransientDataStore {
    private final File _dbDir;
    private final KademliaNetworkDatabaseFacade _facade;
    private final Writer _writer;
    private final ReadJob _readJob;
    private volatile boolean _initialized;
    private final boolean _flat;
    private final int _networkID;
    
    private final static int READ_DELAY = 2*60*1000;
    private static final String PROP_FLAT = "router.networkDatabase.flat";
    static final String DIR_PREFIX = "r";
    private static final String B64 = Base64.ALPHABET_I2P;
    
    /**
     *  @param dbDir relative path
     */
    public PersistentDataStore(RouterContext ctx, String dbDir, KademliaNetworkDatabaseFacade facade) throws IOException {
        super(ctx);
        _networkID = ctx.router().getNetworkID();
        _flat = ctx.getBooleanProperty(PROP_FLAT);
        _dbDir = getDbDir(dbDir);
        _facade = facade;
        _readJob = new ReadJob();
        _context.jobQueue().addJob(_readJob);
        ctx.statManager().createRateStat("netDb.writeClobber", "How often we clobber a pending netDb write", "NetworkDatabase", new long[] { 20*60*1000 });
        ctx.statManager().createRateStat("netDb.writePending", "How many pending writes are there", "NetworkDatabase", new long[] { 60*1000 });
        ctx.statManager().createRateStat("netDb.writeOut", "How many we wrote", "NetworkDatabase", new long[] { 20*60*1000 });
        ctx.statManager().createRateStat("netDb.writeTime", "How long it took", "NetworkDatabase", new long[] { 20*60*1000 });
        //ctx.statManager().createRateStat("netDb.readTime", "How long one took", "NetworkDatabase", new long[] { 20*60*1000 });
        _writer = new Writer();
        I2PThread writer = new I2PThread(_writer, "DBWriter");
        // stop() must be called to flush data to disk
        //writer.setDaemon(true);
        writer.start();
    }

    @Override
    public boolean isInitialized() { return _initialized; }

    // this doesn't stop the read job or the writer, maybe it should?
    @Override
    public void stop() {
        super.stop();
        _writer.flush();
    }
    
    @Override
    public void restart() {
        super.restart();
    }
    
    @Override
    public void rescan() {
        if (_initialized)
            _readJob.wakeup();
    }

    @Override
    public DatabaseEntry get(Hash key) {
        return get(key, true);
    }

    /**
     *  Prepare for having only a partial set in memory and the rest on disk
     *  @param persist if false, call super only, don't access disk
     */
    @Override
    public DatabaseEntry get(Hash key, boolean persist) {
        DatabaseEntry rv =  super.get(key);
/*****
        if (rv != null || !persist)
            return rv;
        rv = _writer.get(key);
        if (rv != null)
            return rv;
        Job rrj = new ReadRouterJob(getRouterInfoName(key), key));
         run in same thread
        rrj.runJob();
*******/    
        return rv;
    }

    @Override
    public DatabaseEntry remove(Hash key) {
        return remove(key, true);
    }

    /*
     *  @param persist if false, call super only, don't access disk
     */
    @Override
    public DatabaseEntry remove(Hash key, boolean persist) {
        if (persist) {
            _writer.remove(key);
            _context.jobQueue().addJob(new RemoveJob(key));
        }
        return super.remove(key);
    }
    
    @Override
    public boolean put(Hash key, DatabaseEntry data) {
        return put(key, data, true);
    }

    /*
     *  @param persist if false, call super only, don't access disk
     *  @return success
     */
    @Override
    public boolean put(Hash key, DatabaseEntry data, boolean persist) {
        if ( (data == null) || (key == null) ) return false;
        boolean rv = super.put(key, data);
        // Don't bother writing LeaseSets to disk
        if (rv && persist && data.getType() == DatabaseEntry.KEY_TYPE_ROUTERINFO)
            _writer.queue(key, data);
        return rv;
    }
    
    private class RemoveJob extends JobImpl {
        private final Hash _key;
        public RemoveJob(Hash key) {
            super(PersistentDataStore.this._context);
            _key = key;
        }
        public String getName() { return "Delete RI file"; }
        public void runJob() {
            if (_log.shouldLog(Log.INFO))
                _log.info("Removing key " + _key /* , getAddedBy() */);
            try {
                removeFile(_key, _dbDir);
            } catch (IOException ioe) {
                _log.error("Error removing key " + _key, ioe);
            }
        }
    }
    
    /** How many files to write every 10 minutes. Doesn't make sense to limit it,
     *  they just back up in the queue hogging memory.
     */
    private static final int WRITE_LIMIT = 10000;
    private static final long WRITE_DELAY = 10*60*1000;

    /*
     * Queue up writes, write unlimited files every 10 minutes.
     * Since we write all we have, don't save the write order.
     * We store a reference to the data here too,
     * rather than simply pull it from super.get(), because
     * we will soon have to implement a scheme for keeping only
     * a subset of all DatabaseEntrys in memory and keeping the rest on disk.
     */
    private class Writer implements Runnable, Flushable {
        private final Map<Hash, DatabaseEntry>_keys;
        private final Object _waitLock;
        private volatile boolean _quit;

        public Writer() { 
            _keys = new ConcurrentHashMap<Hash, DatabaseEntry>(64);
            _waitLock = new Object();
        }

        public void queue(Hash key, DatabaseEntry data) {
            int pending = _keys.size();
            boolean exists = (null != _keys.put(key, data));
            if (exists)
                _context.statManager().addRateData("netDb.writeClobber", pending);
            _context.statManager().addRateData("netDb.writePending", pending);
        }

        public void remove(Hash key) {
            _keys.remove(key);
        }

        public void run() {
            _quit = false;
            Hash key = null;
            DatabaseEntry data = null;
            int count = 0;
            int lastCount = 0;
            long startTime = 0;
            while (true) {
                // get a new iterator every time to get a random entry without
                // having concurrency issues or copying to a List or Array
                Iterator<Map.Entry<Hash, DatabaseEntry>> iter = _keys.entrySet().iterator();
                try {
                    Map.Entry<Hash, DatabaseEntry> entry = iter.next();
                    key = entry.getKey();
                    data = entry.getValue();
                    iter.remove();
                    count++;
                } catch (NoSuchElementException nsee) {
                    lastCount = count;
                    count = 0;
                } catch (IllegalStateException ise) {
                    lastCount = count;
                    count = 0;
                }

                if (key != null) {
                    if (data != null) {
                        // synch with the reader job
                        synchronized (_dbDir) {
                            write(key, data);
                        }
                        data = null;
                    }
                    key = null;
                }
                if (count >= WRITE_LIMIT)
                    count = 0;
                if (count == 0) {
                    if (lastCount > 0) {
                        long time = _context.clock().now() - startTime;
                        if (_log.shouldLog(Log.INFO))
                            _log.info("Wrote " + lastCount + " entries to disk in " + time);
                         _context.statManager().addRateData("netDb.writeOut", lastCount);
                         _context.statManager().addRateData("netDb.writeTime", time);
                    }
                    if (_quit)
                        break;
                    synchronized (_waitLock) {
                        try {
                            _waitLock.wait(WRITE_DELAY);
                        } catch (InterruptedException ie) {}
                    }
                    startTime = _context.clock().now();
                }
            }
        }

        public void flush() {
            synchronized(_waitLock) {
                _quit = true;
                _waitLock.notifyAll();
            }
        }
    }
    
    private void write(Hash key, DatabaseEntry data) {
        if (_log.shouldLog(Log.INFO))
            _log.info("Writing key " + key);
        OutputStream fos = null;
        File dbFile = null;
        try {
            String filename = null;

            if (data.getType() == DatabaseEntry.KEY_TYPE_LEASESET)
                filename = getLeaseSetName(key);
            else if (data.getType() == DatabaseEntry.KEY_TYPE_ROUTERINFO)
                filename = getRouterInfoName(key);
            else
                throw new IOException("We don't know how to write objects of type " + data.getClass().getName());

            dbFile = new File(_dbDir, filename);
            long dataPublishDate = getPublishDate(data);
            if (dbFile.lastModified() < dataPublishDate) {
                // our filesystem is out of date, lets replace it
                fos = new SecureFileOutputStream(dbFile);
                fos = new BufferedOutputStream(fos);
                try {
                    data.writeBytes(fos);
                    fos.close();
                    dbFile.setLastModified(dataPublishDate);
                } catch (DataFormatException dfe) {
                    _log.error("Error writing out malformed object as " + key + ": " 
                               + data, dfe);
                    dbFile.delete();
                }
            } else {
                // we've already written the file, no need to waste our time
                if (_log.shouldLog(Log.DEBUG))
                    _log.debug("Not writing " + key.toBase64() + ", as its up to date on disk (file mod-publish=" +
                               (dbFile.lastModified()-dataPublishDate) + ")");
            }
        } catch (IOException ioe) {
            _log.error("Error writing out the object", ioe);
        } finally {
            if (fos != null) try { fos.close(); } catch (IOException ioe) {}
        }
    }
    private long getPublishDate(DatabaseEntry data) {
        return data.getDate();
    }
    
    /**
     *  This was mostly for manual reseeding, i.e. the user manually
     *  copies RI files to the directory. Nobody does this,
     *  so this is run way too often.
     *
     *  But it's also for migrating and reading the files after a reseed.
     *  Reseed task calls wakeup() on completion.
     *  As of 0.9.4, also initiates an automatic reseed if necessary.
     */
    private class ReadJob extends JobImpl {
        private volatile long _lastModified;
        private volatile long _lastReseed;
        private static final int MIN_ROUTERS = KademliaNetworkDatabaseFacade.MIN_RESEED;
        private static final long MIN_RESEED_INTERVAL = 90*60*1000;

        public ReadJob() {
            super(PersistentDataStore.this._context);
        }

        public String getName() { return "DB Read Job"; }

        public void runJob() {
            if (getContext().router().gracefulShutdownInProgress()) {
                // don't cause more disk I/O while saving,
                // or start a reseed
                requeue(READ_DELAY);
                return;
            }
            long now = getContext().clock().now();
            // check directory mod time to save a lot of object churn in scanning all the file names
            long lastMod = _dbDir.lastModified();
            // if size() (= RI + LS) is too low, call anyway to check for reseed
            boolean shouldScan = lastMod > _lastModified || size() < MIN_ROUTERS + 10;
            if (!shouldScan && !_flat) {
                for (int j = 0; j < B64.length(); j++) {
                    File subdir = new File(_dbDir, DIR_PREFIX + B64.charAt(j));
                    if (subdir.lastModified() > _lastModified) {
                        shouldScan = true;
                        break;
                    }
                }
            }
            if (shouldScan) {
                _log.info("Rereading new files");
                // synch with the writer job
                synchronized (_dbDir) {
                    // _lastModified must be 0 for the first run
                    readFiles();
                }
                _lastModified = now;
            }
            requeue(READ_DELAY);
        }
        
        public void wakeup() {
            requeue(0);
        }
        
        private void readFiles() {
            int routerCount = 0;

            File routerInfoFiles[] = _dbDir.listFiles(RouterInfoFilter.getInstance());
            if (_flat) {
                if (routerInfoFiles != null) {
                    routerCount = routerInfoFiles.length;
                    for (int i = 0; i < routerInfoFiles.length; i++) {
                        // drop out if the router gets killed right after startup
                        if (!_context.router().isAlive())
                            break;
                        Hash key = getRouterInfoHash(routerInfoFiles[i].getName());
                        if ( (key != null) && (!isKnown(key)) ) {
                            // Run it inline so we don't clog up the job queue, esp. at startup
                            // Also this allows us to wait until it is really done to call checkReseed() and set _initialized
                            //PersistentDataStore.this._context.jobQueue().addJob(new ReadRouterJob(routerInfoFiles[i], key));
                            //long start = System.currentTimeMillis();
                            (new ReadRouterJob(routerInfoFiles[i], key)).runJob();
                            //_context.statManager().addRateData("netDb.readTime", System.currentTimeMillis() - start);
                        }
                    }
                }
            } else {
                // move all new RIs to subdirs, then scan those
                if (routerInfoFiles != null)
                    migrate(_dbDir, routerInfoFiles);
                // Loading the files in-order causes clumping in the kbuckets,
                // and bias on early peer selection, so first collect all the files,
                // then shuffle and load.
                List<File> toRead = new ArrayList<File>(2048);
                for (int j = 0; j < B64.length(); j++) {
                    File subdir = new File(_dbDir, DIR_PREFIX + B64.charAt(j));
                    File[] files = subdir.listFiles(RouterInfoFilter.getInstance());
                    if (files == null)
                        continue;
                    long lastMod = subdir.lastModified();
                    if (routerCount >= MIN_ROUTERS && lastMod <= _lastModified)
                        continue;
                    routerCount += files.length;
                    if (lastMod <= _lastModified)
                        continue;
                    for (int i = 0; i < files.length; i++) {
                        toRead.add(files[i]);
                    }
                }
                Collections.shuffle(toRead, _context.random());
                for (File file : toRead) {
                    Hash key = getRouterInfoHash(file.getName());
                    if (key != null && !isKnown(key))
                        (new ReadRouterJob(file, key)).runJob();
                }
            }
            
            if (!_initialized) {
                _initialized = true;
                if (_facade.reseedChecker().checkReseed(routerCount)) {
                    _lastReseed = _context.clock().now();
                    // checkReseed will call wakeup() when done and we will run again
                } else {
                    _context.router().setNetDbReady();
                }
            } else if (_lastReseed < _context.clock().now() - MIN_RESEED_INTERVAL) {
                int count = Math.min(routerCount, size());
                if (count < MIN_ROUTERS) {
                    if (_facade.reseedChecker().checkReseed(count))
                        _lastReseed = _context.clock().now();
                        // checkReseed will call wakeup() when done and we will run again
                } else {
                    _context.router().setNetDbReady();
                }
            }
        }
    }
    
    private class ReadRouterJob extends JobImpl {
        private final File _routerFile;
        private final Hash _key;
        private long _knownDate;

        /**
         *  @param key must match the RI hash in the file
         */
        public ReadRouterJob(File routerFile, Hash key) {
            super(PersistentDataStore.this._context);
            _routerFile = routerFile;
            _key = key;
        }

        public String getName() { return "Read RouterInfo"; }
        
        private boolean shouldRead() {
            // persist = false to call only super.get()
            DatabaseEntry data = get(_key, false);
            if (data == null) return true;
            if (data.getType() == DatabaseEntry.KEY_TYPE_ROUTERINFO) {
                _knownDate = ((RouterInfo)data).getPublished();
                long fileDate = _routerFile.lastModified();
                // don't overwrite recent netdb RIs with reseed data
                return fileDate > _knownDate + (60*60*1000);
            } else {
                // safety measure - prevent injection from reseeding
                _log.error("Prevented LS overwrite by RI " + _key + " from " + _routerFile);
                return false;
            }
        }

        public void runJob() {
            if (!shouldRead()) return;
            if (_log.shouldLog(Log.DEBUG))
                _log.debug("Reading " + _routerFile);

                InputStream fis = null;
                boolean corrupt = false;
                try {
                    fis = new FileInputStream(_routerFile);
                    fis = new BufferedInputStream(fis);
                    RouterInfo ri = new RouterInfo();
                    ri.readBytes(fis, true);  // true = verify sig on read
                    if (ri.getNetworkId() != _networkID) {
                        corrupt = true;
                        if (_log.shouldLog(Log.ERROR))
                            _log.error("The router "
                                       + ri.getIdentity().calculateHash().toBase64() 
                                       + " is from a different network");
                    } else if (!ri.getIdentity().calculateHash().equals(_key)) {
                        // prevent injection from reseeding
                        // this is checked in KNDF.validate() but catch it sooner and log as error.
                        corrupt = true;
                        if (_log.shouldLog(Log.WARN))
                            _log.warn(ri.getIdentity().calculateHash() + " does not match " + _key + " from " + _routerFile);
                    } else if (ri.getPublished() <= _knownDate) {
                        // Don't store but don't delete
                        if (_log.shouldLog(Log.WARN))
                            _log.warn("Skipping since netdb newer than " + _routerFile);
                    } else if (getContext().blocklist().isBlocklisted(ri)) {
                        corrupt = true;
                        if (_log.shouldLog(Log.WARN))
                            _log.warn(ri.getHash() + " is blocklisted");
                    } else {
                        try {
                            // persist = false so we don't write what we just read
                            _facade.store(ri.getIdentity().getHash(), ri, false);
                            // when heardAbout() was removed from TransientDataStore, it broke
                            // profile bootstrapping for new routers,
                            // so add it here.
                            getContext().profileManager().heardAbout(ri.getIdentity().getHash(), ri.getPublished());
                        } catch (IllegalArgumentException iae) {
                            if (_log.shouldLog(Log.INFO))
                                _log.info("Refused locally loaded routerInfo - deleting", iae);
                            corrupt = true;
                        }
                    }
                } catch (DataFormatException dfe) {
                    if (_log.shouldLog(Log.INFO))
                        _log.info("Error reading the routerInfo from " + _routerFile.getName(), dfe);
                    corrupt = true;
                } catch (IOException ioe) {
                    if (_log.shouldLog(Log.INFO))
                        _log.info("Unable to read the router reference in " + _routerFile.getName(), ioe);
                    corrupt = true;
                } catch (RuntimeException e) {
                    // key certificate problems, etc., don't let one bad RI kill the whole thing
                    if (_log.shouldLog(Log.INFO))
                        _log.info("Unable to read the router reference in " + _routerFile.getName(), e);
                    corrupt = true;
                } finally {
                    if (fis != null) try { fis.close(); } catch (IOException ioe) {}
                }
                if (corrupt) _routerFile.delete();
        }
    }
    
    
    private File getDbDir(String dbDir) throws IOException {
        File f = new SecureDirectory(_context.getRouterDir(), dbDir);
        if (!f.exists()) {
            boolean created = f.mkdirs();
            if (!created)
                throw new IOException("Unable to create the DB directory [" + f.getAbsolutePath() + "]");
        }
        if (!f.isDirectory())
            throw new IOException("DB directory [" + f.getAbsolutePath() + "] is not a directory!");
        if (!f.canRead())
            throw new IOException("DB directory [" + f.getAbsolutePath() + "] is not readable!");
        if (!f.canWrite())
            throw new IOException("DB directory [" + f.getAbsolutePath() + "] is not writable!");
        if (_flat) {
            unmigrate(f);
        } else {
            for (int j = 0; j < B64.length(); j++) {
                File subdir = new SecureDirectory(f, DIR_PREFIX + B64.charAt(j));
                if (!subdir.exists())
                    subdir.mkdir();
            }
            File routerInfoFiles[] = f.listFiles(RouterInfoFilter.getInstance());
            if (routerInfoFiles != null)
                migrate(f, routerInfoFiles);
        }
        return f;
    }

    /**
     *  Migrate from two-level to one-level directory structure
     *  @since 0.9.5
     */
    private static void unmigrate(File dbdir) {
        for (int j = 0; j < B64.length(); j++) {
            File subdir = new File(dbdir, DIR_PREFIX + B64.charAt(j));
            File[] files = subdir.listFiles(RouterInfoFilter.getInstance());
            if (files == null)
                continue;
            for (int i = 0; i < files.length; i++) {
                File from = files[i];
                File to = new File(dbdir, from.getName());
                FileUtil.rename(from, to);
            }
        }
    }

    /**
     *  Migrate from one-level to two-level directory structure
     *  @since 0.9.5
     */
    private static void migrate(File dbdir, File[] files) {
        for (int i = 0; i < files.length; i++) {
            File from = files[i];
            if (!from.isFile())
                continue;
            File dir = new File(dbdir, DIR_PREFIX + from.getName().charAt(ROUTERINFO_PREFIX.length()));
            File to = new File(dir, from.getName());
            FileUtil.rename(from, to);
        }
    }
    
    private final static String LEASESET_PREFIX = "leaseSet-";
    private final static String LEASESET_SUFFIX = ".dat";
    private final static String ROUTERINFO_PREFIX = "routerInfo-";
    private final static String ROUTERINFO_SUFFIX = ".dat";
    
    private static String getLeaseSetName(Hash hash) {
        return LEASESET_PREFIX + hash.toBase64() + LEASESET_SUFFIX;
    }

    private String getRouterInfoName(Hash hash) {
        String b64 = hash.toBase64();
        if (_flat)
            return ROUTERINFO_PREFIX + b64 + ROUTERINFO_SUFFIX;
        return DIR_PREFIX + b64.charAt(0) + File.separatorChar + ROUTERINFO_PREFIX + b64 + ROUTERINFO_SUFFIX;
    }

    /**
     *  The persistent RI file for a hash.
     *  This is available before the netdb subsystem is running, so we can delete our old RI.
     *
     *  @return non-null, should be absolute, does not necessarily exist
     *  @since 0.9.23
     */
    public static File getRouterInfoFile(RouterContext ctx, Hash hash) {
        String b64 = hash.toBase64();
        File dir = new File(ctx.getRouterDir(), ctx.getProperty(KademliaNetworkDatabaseFacade.PROP_DB_DIR, KademliaNetworkDatabaseFacade.DEFAULT_DB_DIR));
        if (ctx.getBooleanProperty(PROP_FLAT))
            return new File(dir, ROUTERINFO_PREFIX + b64 + ROUTERINFO_SUFFIX);
        return new File(dir, DIR_PREFIX + b64.charAt(0) + File.separatorChar + ROUTERINFO_PREFIX + b64 + ROUTERINFO_SUFFIX);
    }
    
    /**
     *  Package private for installer BundleRouterInfos
     */
    static Hash getRouterInfoHash(String filename) {
        return getHash(filename, ROUTERINFO_PREFIX, ROUTERINFO_SUFFIX);
    }
    
    private static Hash getHash(String filename, String prefix, String suffix) {
        try {
            String key = filename.substring(prefix.length());
            key = key.substring(0, key.length() - suffix.length());
            //Hash h = new Hash();
            //h.fromBase64(key);
            byte[] b = Base64.decode(key);
            if (b == null)
                return null;
            Hash h = Hash.create(b);
            return h;
        } catch (RuntimeException e) {
            // static
            //_log.warn("Unable to fetch the key from [" + filename + "]", e);
            return null;
        }
    }
    
    private void removeFile(Hash key, File dir) throws IOException {
        String riName = getRouterInfoName(key);
        File f = new File(dir, riName);
        if (f.exists()) {
            boolean removed = f.delete();
            if (!removed) {
                if (_log.shouldLog(Log.WARN))
                    _log.warn("Unable to remove router info at " + f.getAbsolutePath());
            } else if (_log.shouldLog(Log.INFO)) {
                _log.info("Removed router info at " + f.getAbsolutePath());
            }
            return;
        }
    }
    
    static class RouterInfoFilter implements FilenameFilter {
        private static final FilenameFilter _instance = new RouterInfoFilter();
        public static final FilenameFilter getInstance() { return _instance; }
        public boolean accept(File dir, String name) {
            if (name == null) return false;
            name = name.toUpperCase(Locale.US);
            return (name.startsWith(ROUTERINFO_PREFIX.toUpperCase(Locale.US)) && name.endsWith(ROUTERINFO_SUFFIX.toUpperCase(Locale.US)));
        }
    }
}
