load_form.h

Go to the documentation of this file.

/* Copyright, 2001 Nevrax Ltd.
 *
 * This file is part of NEVRAX NEL.
 * NEVRAX NEL is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2, or (at your option)
 * any later version.

 * NEVRAX NEL is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * General Public License for more details.

 * You should have received a copy of the GNU General Public License
 * along with NEVRAX NEL; see the file COPYING. If not, write to the
 * Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston,
 * MA 02111-1307, USA.
 */

#ifndef NL_LOAD_FORM_H
#define NL_LOAD_FORM_H

#include "nel/misc/types_nl.h"

#include <map>
#include <string>
#include <vector>

#include "nel/misc/path.h"
#include "nel/misc/file.h"
#include "nel/misc/sheet_id.h"
#include "nel/misc/algo.h"

#include "u_form_loader.h"
#include "u_form.h"


/*
struct TLoadFormDicoEntry
{
    std::string     Filename;
    uint32          ModificationDate;

    void serial(NLMISC::IStream &s)
    {
        s.serial(Filename);
        s.serial(ModificationDate);
    }
};
*/
const uint32        PACKED_SHEET_HEADER = 'PKSH';
const uint32        PACKED_SHEET_VERSION = 5;
// This Version may be used if you want to use the serialVersion() system in loadForm()
const uint32        PACKED_SHEET_VERSION_COMPATIBLE = 0;

// ***************************************************************************
template <class T>
void loadForm (const std::vector<std::string> &sheetFilters, const std::string &packedFilename, std::map<NLMISC::CSheetId, T> &container, bool updatePackedSheet=true, bool errorIfPackedSheetNotGood=true)
{
    std::vector<std::string>                        dictionnary;
    std::map<std::string, uint>                     dictionnaryIndex;
    std::map<NLMISC::CSheetId, std::vector<uint32> >    dependencies;
    std::vector<uint32>                             dependencyDates;

    // check the extension (i know that file like "foo.packed_sheetsbar" will be accepted but this check is enough...)
    nlassert (packedFilename.find (".packed_sheets") != std::string::npos);

    std::string packedFilenamePath = NLMISC::CPath::lookup(NLMISC::CFile::getFilename(packedFilename), false, false);
    if (packedFilenamePath.empty())
    {
        packedFilenamePath = packedFilename;
    }

    // make sure the CSheetId singleton has been properly initialised
    NLMISC::CSheetId::init(updatePackedSheet);

    // load the packed sheet if exists
    try
    {
        NLMISC::CIFile ifile;
        ifile.setCacheFileOnOpen(true);
        if (!ifile.open (packedFilenamePath))
        {
            throw   NLMISC::Exception("can't open PackedSheet %s", packedFilenamePath.c_str());
        }
        // an exception will be launch if the file is not the good version or if the file is not found

        nlinfo ("loadForm(): Loading packed file '%s'", packedFilename.c_str());

        // read the header
        ifile.serialCheck(PACKED_SHEET_HEADER);
        ifile.serialCheck(PACKED_SHEET_VERSION);
        ifile.serialVersion(PACKED_SHEET_VERSION_COMPATIBLE);

        // Read depend block size
        uint32  dependBlockSize;
        ifile.serial(dependBlockSize);

        // Read the dependencies only if update packed sheet
        if(updatePackedSheet)
        {
            // read the dictionnary
            {
                ifile.serialCont(dictionnary);
            }
            // read the dependency data
            {
                uint32 depSize;
                ifile.serial(depSize);
                for (uint i=0; i<depSize; ++i)
                {
                    NLMISC::CSheetId sheetId;

                    // Avoid copy, use []
                    ifile.serial(sheetId);
                    ifile.serialCont(dependencies[sheetId]);
                }
            }
        }
        // else dummy read one big block => no heavy reallocation / free
        else if(dependBlockSize>0)
        {
            std::vector<uint8>  bigBlock;
            bigBlock.resize(dependBlockSize);
            ifile.serialBuffer(&bigBlock[0], dependBlockSize);
        }

        // read the packed sheet data
        uint32  nbEntries;
        uint32  ver;
        ifile.serial (nbEntries);
        ifile.serial (ver);
        if(ver != T::getVersion ())
            throw NLMISC::Exception("The packed sheet version in stream is different of the code");
        ifile.serialCont (container);
        ifile.close ();
    }
    catch (NLMISC::Exception &e)
    {
        // clear the container because it can contains partially loaded sheet so we must clean it before continue
        container.clear ();
        if (!updatePackedSheet)
        {
            if (errorIfPackedSheetNotGood)
                nlerror ("loadForm(): Exception during reading the packed file and can't reconstruct them (%s)", e.what());
            else
                nlinfo ("loadForm(): Exception during reading the packed file and can't reconstruct them (%s)", e.what());

            return;
        }
        else
        {
            nlinfo ("loadForm(): Exception during reading the packed file, I'll reconstruct it (%s)", e.what());
        }
    }

    // if we don't want to update packed sheet, we have nothing more to do
    if (!updatePackedSheet)
    {
        nlinfo ("Don't update the packed sheet with real sheet");
        return;
    }

    // retreive the date of all dependency file
    {
        for (uint i=0; i<dictionnary.size(); ++i)
        {
            std::string p = NLMISC::CPath::lookup (dictionnary[i], false, false);
            if (!p.empty())
            {
                uint32 d = NLMISC::CFile::getFileModificationDate(p);
                dependencyDates.push_back(d);
            }
            else
            {
                // file not found !
                // write a future date to invalidate any file dependent on it
                nldebug("Can't find dependent file %s !", dictionnary[i].c_str());
                dependencyDates.push_back(0xffffffff);
            }
        }
    }

    // build a vector of the sheetFilters sheet ids (ie: "item")
    std::vector<NLMISC::CSheetId> sheetIds;
    std::vector<std::string> filenames;
    for (uint i = 0; i < sheetFilters.size(); i++)
        NLMISC::CSheetId::buildIdVector(sheetIds, filenames, sheetFilters[i]);

    // if there's no file, nothing to do
    if (sheetIds.empty())
        return;

    // set up the current sheet in container to remove sheet that are in the container and not in the directory anymore
    std::map<NLMISC::CSheetId, bool> sheetToRemove;
    for (typename std::map<NLMISC::CSheetId, T>::iterator it = container.begin(); it != container.end(); it++)
    {
        sheetToRemove.insert (std::make_pair((*it).first, true));
    }

    // check if we need to create a new .pitems or just read it
    uint32 packedFiledate = NLMISC::CFile::getFileModificationDate(packedFilenamePath);

    bool containerChanged = false;

    NLGEORGES::UFormLoader *formLoader = NULL;

    std::vector<uint> NeededToRecompute;

    for (uint k = 0; k < filenames.size(); k++)
    {
        std::string p = NLMISC::CPath::lookup (filenames[k], false, false);
        if (p.empty()) continue;
        uint32 d = NLMISC::CFile::getFileModificationDate(p);

        // no need to remove this sheet
        sheetToRemove[sheetIds[k]] = false;

        if( d > packedFiledate || container.find (sheetIds[k]) == container.end())
        {
            NeededToRecompute.push_back(k);
        }
        else
        {
            // check the date of each parent
            nlassert(dependencies.find(sheetIds[k]) != dependencies.end());
            std::vector<uint32> &depends = dependencies[sheetIds[k]];

            for (uint i=0; i<depends.size(); ++i)
            {
                if (dependencyDates[depends[i]] > packedFiledate)
                {
                    nldebug("Dependency on %s for %s not up to date !",
                        dictionnary[depends[i]].c_str(), sheetIds[k].toString().c_str());
                    NeededToRecompute.push_back(k);
                    break;
                }
            }
        }
    }

    nlinfo ("%d sheets checked, %d need to be recomputed", filenames.size(), NeededToRecompute.size());

    NLMISC::TTime last = NLMISC::CTime::getLocalTime ();
    NLMISC::TTime start = NLMISC::CTime::getLocalTime ();

    NLMISC::CSmartPtr<NLGEORGES::UForm> form;
    std::vector<NLMISC::CSmartPtr<NLGEORGES::UForm> >   cacheFormList;

    for (uint j = 0; j < NeededToRecompute.size(); j++)
    {
        if(NLMISC::CTime::getLocalTime () > last + 5000)
        {
            last = NLMISC::CTime::getLocalTime ();
            if(j>0)
                nlinfo ("%.0f%% completed (%d/%d), %d seconds remaining", (float)j*100.0/NeededToRecompute.size(),j,NeededToRecompute.size(), (NeededToRecompute.size()-j)*(last-start)/j/1000);
        }

        // create the georges loader if necessary
        if (formLoader == NULL)
        {
            NLMISC::WarningLog->addNegativeFilter("CFormLoader: Can't open the form file");
            formLoader = NLGEORGES::UFormLoader::createLoader ();
        }

        //  cache used to retain information (to optimize time).
        if (form)
            cacheFormList.push_back (form);

        // Load the form with given sheet id
        form = formLoader->loadForm (sheetIds[NeededToRecompute[j]].toString().c_str ());
        if (form)
        {
            // build the dependency data
            {
                std::vector<uint32>     depends;
                std::set<std::string>   dependFiles;
                form->getDependencies (dependFiles);
                nlassert(dependFiles.find(sheetIds[NeededToRecompute[j]].toString()) != dependFiles.end());
                // remove the sheet itself from the container
                dependFiles.erase(sheetIds[NeededToRecompute[j]].toString());

                std::set<std::string>::iterator first(dependFiles.begin()), last(dependFiles.end());
                for (; first != last; ++first)
                {
                    const   std::string filename = NLMISC::CFile::getFilename(*first);
                    std::map<std::string,uint>::iterator    findDicIt=dictionnaryIndex.find(filename);

                    if  (findDicIt!=dictionnaryIndex.end())
                    {
                        depends.push_back(findDicIt->second);
                        continue;
                    }

                    std::string p = NLMISC::CPath::lookup (*first, false, false);
                    if  (!p.empty())
                    {
                        uint dicIndex;
                        // add a new dictionnary entry
                        dicIndex = dictionnary.size();
                        dictionnaryIndex.insert(std::make_pair(filename, dictionnary.size()));
                        dictionnary.push_back(filename);

                        // add the dependecy index
                        depends.push_back(dicIndex);
                    }
                }
                // store the dependency list with the sheet ID
                dependencies[sheetIds[NeededToRecompute[j]]] = depends;
            }

            // add the new creature, it could be already loaded by the packed sheets but will be overwritten with the new one
            typedef typename std::map<NLMISC::CSheetId, T>::iterator TType1;
            typedef typename std::pair<TType1, bool> TType2;
            TType2 res = container.insert(std::make_pair(sheetIds[NeededToRecompute[j]],T()));

            (*res.first).second.readGeorges (form, sheetIds[NeededToRecompute[j]]);
            containerChanged = true;
        }
    }

    if(NeededToRecompute.size() > 0)
        nlinfo ("%d seconds to recompute %d sheets", (uint32)(NLMISC::CTime::getLocalTime()-start)/1000, NeededToRecompute.size());

    // free the georges loader if necessary
    if (formLoader != NULL)
    {
        NLGEORGES::UFormLoader::releaseLoader (formLoader);
        NLMISC::WarningLog->removeFilter ("CFormLoader: Can't open the form file");
    }

    // we have now to remove sheets that are in the container and not exist anymore in the sheet directories
    for (std::map<NLMISC::CSheetId, bool>::iterator it2 = sheetToRemove.begin(); it2 != sheetToRemove.end(); it2++)
    {
        if((*it2).second)
        {
            nlinfo ("the sheet '%s' is not in the directory, remove it from container", (*it2).first.toString().c_str());
            container.find((*it2).first)->second.removed();
            container.erase((*it2).first);
            containerChanged = true;
            dependencies.erase((*it2).first);
        }
    }

    // now, save the new container in the packedfile
    try
    {
        if(containerChanged)
        {
            NLMISC::COFile ofile;
            ofile.open(packedFilenamePath);

            // write the header.
            ofile.serialCheck(PACKED_SHEET_HEADER);
            ofile.serialCheck(PACKED_SHEET_VERSION);
            ofile.serialVersion(PACKED_SHEET_VERSION_COMPATIBLE);

            // Write a dummy block size for now
            sint32  posBlockSize= ofile.getPos();
            uint32  dependBlockSize= 0;
            ofile.serial(dependBlockSize);

            // write the dictionnary
            ofile.serialCont(dictionnary);

            // write the dependencies data
            uint32 depSize = dependencies.size();
            ofile.serial(depSize);
            std::map<NLMISC::CSheetId, std::vector<uint32> >::iterator first(dependencies.begin()), last(dependencies.end());
            for (; first != last; ++first)
            {
                NLMISC::CSheetId si = first->first;
                ofile.serial(si);
                ofile.serialCont(first->second);
            }

            // Then get the dictionary + dependencies size, and write it back to posBlockSize
            sint32  endBlockSize= ofile.getPos();
            dependBlockSize= (endBlockSize - posBlockSize) - 4;
            ofile.seek(posBlockSize, NLMISC::IStream::begin);
            ofile.serial(dependBlockSize);
            ofile.seek(endBlockSize, NLMISC::IStream::begin);

            // write the sheet data
            uint32 nbEntries = sheetIds.size();
            uint32 ver = T::getVersion ();
            ofile.serial (nbEntries);
            ofile.serial (ver);
            ofile.serialCont(container);
            ofile.close ();
        }
    }
    catch (NLMISC::Exception &e)
    {
        nlinfo ("loadForm(): Exception during saving the packed file, it will be recreated next launch (%s)", e.what());
    }

    // housekeeping
    sheetIds.clear ();
    filenames.clear ();
}

// ***************************************************************************
template <class T>
void loadForm (const std::string &sheetFilter, const std::string &packedFilename, std::map<NLMISC::CSheetId, T> &container, bool updatePackedSheet=true, bool errorIfPackedSheetNotGood=true)
{
    std::vector<std::string> vs;
    vs.push_back(sheetFilter);
    loadForm(vs, packedFilename, container, updatePackedSheet, errorIfPackedSheetNotGood);
}


// ***************************************************************************
// variant with smart pointers, maintain with function above
template <class T>
void loadForm2(const std::vector<std::string> &sheetFilters, const std::string &packedFilename, std::map<NLMISC::CSheetId, NLMISC::CSmartPtr<T> > &container, bool updatePackedSheet=true, bool errorIfPackedSheetNotGood=true)
{
    std::vector<std::string>                        dictionnary;
    std::map<std::string, uint>                     dictionnaryIndex;
    std::map<NLMISC::CSheetId, std::vector<uint32> >    dependencies;
    std::vector<uint32>                             dependencyDates;

    // check the extension (i know that file like "foo.packed_sheetsbar" will be accepted but this check is enough...)
    nlassert (packedFilename.find (".packed_sheets") != std::string::npos);

    std::string packedFilenamePath = NLMISC::CPath::lookup(NLMISC::CFile::getFilename(packedFilename), false, false);
    if (packedFilenamePath.empty())
    {
        packedFilenamePath = packedFilename;
    }

    // make sure the CSheetId singleton has been properly initialised
    NLMISC::CSheetId::init(updatePackedSheet);

    // load the packed sheet if exists
    try
    {
        NLMISC::CIFile ifile;
        ifile.setCacheFileOnOpen(true);
        if (!ifile.open (packedFilenamePath))
        {
            throw   NLMISC::Exception("can't open PackedSheet %s", packedFilenamePath.c_str());
        }
        // an exception will be launch if the file is not the good version or if the file is not found

        nlinfo ("loadForm(): Loading packed file '%s'", packedFilename.c_str());

        // read the header
        ifile.serialCheck(PACKED_SHEET_HEADER);
        ifile.serialCheck(PACKED_SHEET_VERSION);
        sint    loadFormVersion= ifile.serialVersion(PACKED_SHEET_VERSION_COMPATIBLE);

        // Read depend block size
        uint32  dependBlockSize;
        ifile.serial(dependBlockSize);

        // Read the dependencies only if update packed sheet
        if(updatePackedSheet)
        {
            // read the dictionnary
            {
                ifile.serialCont(dictionnary);
            }
            // read the dependency data
            {
                uint32 depSize;
                ifile.serial(depSize);
                for (uint i=0; i<depSize; ++i)
                {
                    NLMISC::CSheetId sheetId;

                    // Avoid copy, use []
                    ifile.serial(sheetId);
                    ifile.serialCont(dependencies[sheetId]);
                }
            }
        }
        // else dummy read one big block => no heavy reallocation / free
        else if(dependBlockSize>0)
        {
            std::vector<uint8>  bigBlock;
            bigBlock.resize(dependBlockSize);
            ifile.serialBuffer(&bigBlock[0], dependBlockSize);
        }

        // read the packed sheet data
        uint32  nbEntries;
        uint32  ver;
        ifile.serial (nbEntries);
        ifile.serial (ver);
        if(ver != T::getVersion ())
            throw NLMISC::Exception("The packed sheet version in stream is different of the code");
        ifile.serialPtrCont (container);
        ifile.close ();
    }
    catch (NLMISC::Exception &e)
    {
        // clear the container because it can contains partially loaded sheet so we must clean it before continue
        container.clear ();
        if (!updatePackedSheet)
        {
            if (errorIfPackedSheetNotGood)
                nlerror ("loadForm(): Exception during reading the packed file and can't reconstruct them (%s)", e.what());
            else
                nlinfo ("loadForm(): Exception during reading the packed file and can't reconstruct them (%s)", e.what());

            return;
        }
        else
        {
            nlinfo ("loadForm(): Exception during reading the packed file, I'll reconstruct it (%s)", e.what());
        }
    }

    // if we don't want to update packed sheet, we have nothing more to do
    if (!updatePackedSheet)
    {
        nlinfo ("Don't update the packed sheet with real sheet");
        return;
    }

    // retreive the date of all dependency file
    {
        for (uint i=0; i<dictionnary.size(); ++i)
        {
            std::string p = NLMISC::CPath::lookup (dictionnary[i], false, false);
            if (!p.empty())
            {
                uint32 d = NLMISC::CFile::getFileModificationDate(p);
                dependencyDates.push_back(d);
            }
            else
            {
                // file not found !
                // write a future date to invalidate any file dependent on it
                nldebug("Can't find dependent file %s !", dictionnary[i].c_str());
                dependencyDates.push_back(0xffffffff);
            }
        }
    }

    // build a vector of the sheetFilters sheet ids (ie: "item")
    std::vector<NLMISC::CSheetId> sheetIds;
    std::vector<std::string> filenames;
    for (uint i = 0; i < sheetFilters.size(); i++)
        NLMISC::CSheetId::buildIdVector(sheetIds, filenames, sheetFilters[i]);

    // if there's no file, nothing to do
    if (sheetIds.empty())
        return;

    // set up the current sheet in container to remove sheets that are in the container and not in the directory anymore
    std::map<NLMISC::CSheetId, bool> sheetToRemove;
    for (typename std::map<NLMISC::CSheetId, NLMISC::CSmartPtr<T> >::iterator it = container.begin(); it != container.end(); it++)
    {
        sheetToRemove.insert (std::make_pair((*it).first, true));
    }

    // check if we need to create a new .pitems or just read it
    uint32 packedFiledate = NLMISC::CFile::getFileModificationDate(packedFilenamePath);

    bool containerChanged = false;

    NLGEORGES::UFormLoader *formLoader = NULL;

    std::vector<uint> NeededToRecompute;

    for (uint k = 0; k < filenames.size(); k++)
    {
        std::string p = NLMISC::CPath::lookup (filenames[k], false, false);
        if (p.empty()) continue;
        uint32 d = NLMISC::CFile::getFileModificationDate(p);

        // no need to remove this sheet
        sheetToRemove[sheetIds[k]] = false;

        if( d > packedFiledate || container.find (sheetIds[k]) == container.end())
        {
            NeededToRecompute.push_back(k);
        }
        else
        {
            // check the date of each parent
            nlassert(dependencies.find(sheetIds[k]) != dependencies.end());
            std::vector<uint32> &depends = dependencies[sheetIds[k]];

            for (uint i=0; i<depends.size(); ++i)
            {
                if (dependencyDates[depends[i]] > packedFiledate)
                {
                    nldebug("Dependency on %s for %s not up to date !",
                        dictionnary[depends[i]].c_str(), sheetIds[k].toString().c_str());
                    NeededToRecompute.push_back(k);
                    break;
                }
            }
        }
    }

    nlinfo ("%d sheets checked, %d need to be recomputed", filenames.size(), NeededToRecompute.size());

    NLMISC::TTime last = NLMISC::CTime::getLocalTime ();
    NLMISC::TTime start = NLMISC::CTime::getLocalTime ();

    NLMISC::CSmartPtr<NLGEORGES::UForm> form;
    std::vector<NLMISC::CSmartPtr<NLGEORGES::UForm> >   cacheFormList;

    for (uint j = 0; j < NeededToRecompute.size(); j++)
    {
        if(NLMISC::CTime::getLocalTime () > last + 5000)
        {
            last = NLMISC::CTime::getLocalTime ();
            if(j>0)
                nlinfo ("%.0f%% completed (%d/%d), %d seconds remaining", (float)j*100.0/NeededToRecompute.size(),j,NeededToRecompute.size(), (NeededToRecompute.size()-j)*(last-start)/j/1000);
        }

        // create the georges loader if necessary
        if (formLoader == NULL)
        {
            NLMISC::WarningLog->addNegativeFilter("CFormLoader: Can't open the form file");
            formLoader = NLGEORGES::UFormLoader::createLoader ();
        }

        //  cache used to retain information (to optimize time).
        if (form)
            cacheFormList.push_back (form);

        // Load the form with given sheet id
        form = formLoader->loadForm (sheetIds[NeededToRecompute[j]].toString().c_str ());
        if (form)
        {
            // build the dependency data
            {
                std::vector<uint32>     depends;
                std::set<std::string>   dependFiles;
                form->getDependencies (dependFiles);
                nlassert(dependFiles.find(sheetIds[NeededToRecompute[j]].toString()) != dependFiles.end());
                // remove the sheet itself from the container
                dependFiles.erase(sheetIds[NeededToRecompute[j]].toString());

                std::set<std::string>::iterator first(dependFiles.begin()), last(dependFiles.end());
                for (; first != last; ++first)
                {
                    const   std::string filename = NLMISC::CFile::getFilename(*first);
                    std::map<std::string,uint>::iterator    findDicIt=dictionnaryIndex.find(filename);

                    if  (findDicIt!=dictionnaryIndex.end())
                    {
                        depends.push_back(findDicIt->second);
                        continue;
                    }

                    std::string p = NLMISC::CPath::lookup (*first, false, false);
                    if  (!p.empty())
                    {
                        uint dicIndex;
                        // add a new dictionnary entry
                        dicIndex = dictionnary.size();
                        dictionnaryIndex.insert(std::make_pair(filename, dictionnary.size()));
                        dictionnary.push_back(filename);

                        // add the dependency index
                        depends.push_back(dicIndex);
                    }
                }
                // store the dependency list with the sheet ID
                dependencies[sheetIds[NeededToRecompute[j]]] = depends;
            }

            // add the new creature, it could be already loaded by the packed sheets but will be overwritten with the new one
            typedef typename std::map<NLMISC::CSheetId, NLMISC::CSmartPtr<T> >::iterator TType1;
            typedef typename std::pair<TType1, bool> TType2;
            TType2 res = container.insert(std::make_pair(sheetIds[NeededToRecompute[j]], NLMISC::CSmartPtr<T>(new T())));

            (*res.first).second->readGeorges (form, sheetIds[NeededToRecompute[j]]);
            containerChanged = true;
        }
    }

    if(NeededToRecompute.size() > 0)
        nlinfo ("%d seconds to recompute %d sheets", (uint32)(NLMISC::CTime::getLocalTime()-start)/1000, NeededToRecompute.size());

    // free the georges loader if necessary
    if (formLoader != NULL)
    {
        NLGEORGES::UFormLoader::releaseLoader (formLoader);
        NLMISC::WarningLog->removeFilter ("CFormLoader: Can't open the form file");
    }

    // we have now to remove sheets that are in the container and not exist anymore in the sheet directories
    for (std::map<NLMISC::CSheetId, bool>::iterator it2 = sheetToRemove.begin(); it2 != sheetToRemove.end(); it2++)
    {
        if((*it2).second)
        {
            nlinfo ("the sheet '%s' is not in the directory, remove it from container", (*it2).first.toString().c_str());
            container.find((*it2).first)->second->removed();
            container.erase((*it2).first);
            containerChanged = true;
            dependencies.erase((*it2).first);
        }
    }

    // now, save the new container in the packedfile
    try
    {
        if(containerChanged)
        {
            NLMISC::COFile ofile;
            ofile.open(packedFilenamePath);

            // write the header.
            ofile.serialCheck(PACKED_SHEET_HEADER);
            ofile.serialCheck(PACKED_SHEET_VERSION);
            ofile.serialVersion(PACKED_SHEET_VERSION_COMPATIBLE);

            // Write a dummy block size for now
            sint32  posBlockSize= ofile.getPos();
            uint32  dependBlockSize= 0;
            ofile.serial(dependBlockSize);

            // write the dictionnary
            ofile.serialCont(dictionnary);

            // write the dependencies data
            uint32 depSize = dependencies.size();
            ofile.serial(depSize);
            std::map<NLMISC::CSheetId, std::vector<uint32> >::iterator first(dependencies.begin()), last(dependencies.end());
            for (; first != last; ++first)
            {
                NLMISC::CSheetId si = first->first;
                ofile.serial(si);
                ofile.serialCont(first->second);
            }

            // Then get the dicionary + dependencies size, and write it back to posBlockSize
            sint32  endBlockSize= ofile.getPos();
            dependBlockSize= (endBlockSize - posBlockSize) - 4;
            ofile.seek(posBlockSize, NLMISC::IStream::begin);
            ofile.serial(dependBlockSize);
            ofile.seek(endBlockSize, NLMISC::IStream::begin);

            // write the sheet data
            uint32 nbEntries = sheetIds.size();
            uint32 ver = T::getVersion ();
            ofile.serial (nbEntries);
            ofile.serial (ver);
            ofile.serialPtrCont(container);
            ofile.close ();
        }
    }
    catch (NLMISC::Exception &e)
    {
        nlinfo ("loadForm(): Exception during saving the packed file, it will be recreated next launch (%s)", e.what());
    }

    // housekeeping
    sheetIds.clear ();
    filenames.clear ();
}

// ***************************************************************************
// variant with smart pointers, maintain with function above
template <class T>
void loadForm2(const std::string &sheetFilter, const std::string &packedFilename, std::map<NLMISC::CSheetId, T> &container, bool updatePackedSheet=true, bool errorIfPackedSheetNotGood=true)
{
    std::vector<std::string> vs;
    vs.push_back(sheetFilter);
    loadForm2(vs, packedFilename, container, updatePackedSheet, errorIfPackedSheetNotGood);
}



// ***************************************************************************
template <class T>
void loadForm (const std::vector<std::string> &sheetFilters, const std::string &packedFilename, std::map<std::string, T> &container, bool updatePackedSheet=true, bool errorIfPackedSheetNotGood=true)
{
    std::vector<std::string>                        dictionnary;
    std::map<std::string, uint>                     dictionnaryIndex;
    std::map<std::string, std::vector<uint32> >     dependencies;
    std::vector<uint32>                             dependencyDates;

    // check the extension (i know that file like "foo.packed_sheetsbar" will be accepted but this check is enough...)
    nlassert (packedFilename.find (".packed_sheets") != std::string::npos);

    std::string packedFilenamePath = NLMISC::CPath::lookup(packedFilename, false, false);
    if (packedFilenamePath.empty())
    {
        packedFilenamePath = packedFilename;
    }

    // make sure the CSheetId singleton has been properly initialised
//  NLMISC::CSheetId::init(updatePackedSheet);

    // load the packed sheet if exists
    try
    {
        NLMISC::CIFile ifile;
        ifile.setCacheFileOnOpen(true);
        ifile.open (packedFilenamePath);
        // an exception will be launch if the file is not the good version or if the file is not found

        nlinfo ("loadForm(): Loading packed file '%s'", packedFilename.c_str());

        // read the header
        ifile.serialCheck(PACKED_SHEET_HEADER);
        ifile.serialCheck(PACKED_SHEET_VERSION);
        ifile.serialVersion(PACKED_SHEET_VERSION_COMPATIBLE);

        // Read depend block size
        uint32  dependBlockSize;
        ifile.serial(dependBlockSize);

        // Read the dependencies only if update packed sheet
        if(updatePackedSheet)
        {
            // read the dictionnary
            {
                ifile.serialCont(dictionnary);
            }
            // read the dependency data
            {
                uint32 depSize;
                ifile.serial(depSize);
                for (uint i=0; i<depSize; ++i)
                {
                    std::string sheetName;

                    // Avoid copy, use []
                    ifile.serial(sheetName);
                    ifile.serialCont(dependencies[sheetName]);
                }
            }
        }
        // else dummy read one big block => no heavy reallocation / free
        else if(dependBlockSize>0)
        {
            std::vector<uint8>  bigBlock;
            bigBlock.resize(dependBlockSize);
            ifile.serialBuffer(&bigBlock[0], dependBlockSize);
        }

        // read the packed sheet data
        uint32  nbEntries;
        uint32  ver;
        ifile.serial (nbEntries);
        ifile.serial (ver);
        if(ver != T::getVersion ())
            throw NLMISC::Exception("The packed sheet version in stream is different of the code");
        ifile.serialCont (container);
        ifile.close ();
    }
    catch (NLMISC::Exception &e)
    {
        // clear the container because it can contains partially loaded sheet so we must clean it before continue
        container.clear ();
        if (!updatePackedSheet)
        {
            if (errorIfPackedSheetNotGood)
                nlerror ("loadForm(): Exception during reading the packed file and can't reconstruct them (%s)", e.what());
            else
                nlinfo ("loadForm(): Exception during reading the packed file and can't reconstruct them (%s)", e.what());

            return;
        }
        else
        {
            nlinfo ("loadForm(): Exception during reading the packed file, I'll reconstruct it (%s)", e.what());
        }
    }

    // if we don't want to update packed sheet, we have nothing more to do
    if (!updatePackedSheet)
    {
        nlinfo ("Don't update the packed sheet with real sheet");
        return;
    }

    // retreive the date of all dependency file
    {
        for (uint i=0; i<dictionnary.size(); ++i)
        {
            std::string p = NLMISC::CPath::lookup (dictionnary[i], false, false);
            if (!p.empty())
            {
                uint32 d = NLMISC::CFile::getFileModificationDate(p);
                dependencyDates.push_back(d);
            }
            else
            {
                // file not found !
                // write a future date to invalidate any file dependent on it
                nldebug("Can't find dependent file %s !", dictionnary[i].c_str());
                dependencyDates.push_back(0xffffffff);
            }
        }
    }

    // build a vector of the sheetFilters sheet ids (ie: "item")
    std::vector<std::string> sheetNames;
    {
        std::vector<std::string>::const_iterator first(sheetFilters.begin()), last(sheetFilters.end());
        for (; first != last; ++first)
            NLMISC::CPath::getFileList(*first, sheetNames);

    }

    // if there's no file, nothing to do
    if (sheetNames.empty())
        return;

    // set up the current sheet in container to remove sheet that are in the container and not in the directory anymore
    std::map<std::string, bool> sheetToRemove;
    {
        typename std::map<std::string, T>::iterator first(container.begin()), last(container.end());
        for(; first != last; ++first)
            sheetToRemove.insert (make_pair(first->first, true));
    }

    // check if we need to create a new .pitems or just read it
    uint32 packedFiledate = NLMISC::CFile::getFileModificationDate(packedFilenamePath);

    bool containerChanged = false;

    NLGEORGES::UFormLoader *formLoader = NULL;

    std::vector<uint> NeededToRecompute;

    for (uint k = 0; k < sheetNames.size(); k++)
    {
        std::string p = NLMISC::CPath::lookup (sheetNames[k], false, false);
        if (p.empty())
        {
            continue;
        }
        uint32 d = NLMISC::CFile::getFileModificationDate(p);

        // no need to remove this sheet
        sheetToRemove[sheetNames[k]] = false;

        if( d > packedFiledate || container.find (sheetNames[k]) == container.end())
        {
            NeededToRecompute.push_back(k);
        }
        else
        {
            // check the date of each parent
            nlassert(dependencies.find(sheetNames[k]) != dependencies.end());
            std::vector<uint32> &depends = dependencies[sheetNames[k]];

            for (uint i=0; i<depends.size(); ++i)
            {
                if (dependencyDates[depends[i]] > packedFiledate)
                {
                    nldebug("Dependancy on %s for %s not up to date !",
                        dictionnary[depends[i]].c_str(), sheetNames[k].c_str());
                    NeededToRecompute.push_back(k);
                    break;
                }
            }
        }
    }

    nlinfo ("%d sheets checked, %d need to be recomputed", sheetNames.size(), NeededToRecompute.size());

    NLMISC::TTime lastTime = NLMISC::CTime::getLocalTime ();
    NLMISC::TTime start = NLMISC::CTime::getLocalTime ();

    NLMISC::CSmartPtr<NLGEORGES::UForm> form;

    for (uint j = 0; j < NeededToRecompute.size(); j++)
    {
        if(NLMISC::CTime::getLocalTime () > lastTime + 5000)
        {
            lastTime = NLMISC::CTime::getLocalTime ();
            if(j>0)
                nlinfo ("%.0f%% completed (%d/%d), %d seconds remaining", (float)j*100.0/NeededToRecompute.size(),j,NeededToRecompute.size(), (NeededToRecompute.size()-j)*(lastTime-start)/j/1000);
        }

        // create the georges loader if necessary
        if (formLoader == NULL)
        {
            NLMISC::WarningLog->addNegativeFilter("CFormLoader: Can't open the form file");
            formLoader = NLGEORGES::UFormLoader::createLoader ();
        }

        // Load the form with given sheet id
        form = formLoader->loadForm (sheetNames[NeededToRecompute[j]].c_str ());
        if (form)
        {
            // build the dependency data
            {
                std::vector<uint32>     depends;
                std::set<std::string>   dependFiles;
                form->getDependencies (dependFiles);
                nlassert(dependFiles.find(sheetNames[NeededToRecompute[j]]) != dependFiles.end());
                // remove the sheet itself from the container
                dependFiles.erase(sheetNames[NeededToRecompute[j]]);

                std::set<std::string>::iterator first(dependFiles.begin()), last(dependFiles.end());
                for (; first != last; ++first)
                {
                    std::string p = NLMISC::CPath::lookup (*first, false, false);
                    if (!p.empty())
                    {
//                      uint32 date = NLMISC::CFile::getFileModificationDate(p);

                        uint dicIndex;
                        std::string filename = NLMISC::CFile::getFilename(p);

                        if (dictionnaryIndex.find(filename) == dictionnaryIndex.end())
                        {
                            // add a new dictionnary entry
                            dicIndex = dictionnary.size();
                            dictionnaryIndex.insert(std::make_pair(filename, dictionnary.size()));
                            dictionnary.push_back(filename);
                        }
                        else
                        {
                            dicIndex = dictionnaryIndex.find(filename)->second;
                        }

                        // add the dependecy index
                        depends.push_back(dicIndex);
                    }
                }
                // store the dependency list with the sheet ID
                dependencies[sheetNames[NeededToRecompute[j]]] = depends;
            }

            // add the new creature, it could be already loaded by the packed sheets but will be overwritten with the new one
            typedef typename std::map<std::string, T>::iterator TType1;
            typedef typename std::pair<TType1, bool> TType2;
            TType2 res = container.insert(std::make_pair(sheetNames[NeededToRecompute[j]],T()));

            (*res.first).second.readGeorges (form, sheetNames[NeededToRecompute[j]]);
            containerChanged = true;
        }
    }

    nlinfo ("%d seconds to recompute %d sheets", (uint32)(NLMISC::CTime::getLocalTime()-start)/1000, NeededToRecompute.size());

    // free the georges loader if necessary
    if (formLoader != NULL)
    {
        NLGEORGES::UFormLoader::releaseLoader (formLoader);
        NLMISC::WarningLog->removeFilter ("CFormLoader: Can't open the form file");
    }

    // we have now to remove sheet that are in the container and not exist anymore in the sheet directories
    for (std::map<std::string, bool>::iterator it2 = sheetToRemove.begin(); it2 != sheetToRemove.end(); it2++)
    {
        if(it2->second)
        {
            // inform the contained object that it is no more needed.
            container.find(it2->first)->second.removed();
            container.erase(it2->first);
            containerChanged = true;
            dependencies.erase((*it2).first);
        }
    }

    // now, save the new container in the packedfile
    try
    {
        if(containerChanged)
        {
            NLMISC::COFile ofile;
            ofile.open(packedFilenamePath);

            // write the header.
            ofile.serialCheck(PACKED_SHEET_HEADER);
            ofile.serialCheck(PACKED_SHEET_VERSION);
            ofile.serialVersion(PACKED_SHEET_VERSION_COMPATIBLE);

            // Write a dummy block size for now
            sint32  posBlockSize= ofile.getPos();
            uint32  dependBlockSize= 0;
            ofile.serial(dependBlockSize);

            // write the dictionnary
            ofile.serialCont(dictionnary);

            // write the dependencies data
            uint32 depSize = dependencies.size();
            ofile.serial(depSize);
            std::map<std::string, std::vector<uint32> >::iterator first(dependencies.begin()), last(dependencies.end());
            for (; first != last; ++first)
            {
                std::string  sheetName = first->first;
                ofile.serial(sheetName);
                ofile.serialCont(first->second);
            }

            // Then get the dicionary + dependencies size, and write it back to posBlockSize
            sint32  endBlockSize= ofile.getPos();
            dependBlockSize= (endBlockSize - posBlockSize) - 4;
            ofile.seek(posBlockSize, NLMISC::IStream::begin);
            ofile.serial(dependBlockSize);
            ofile.seek(endBlockSize, NLMISC::IStream::begin);

            // write the sheet data
            uint32 nbEntries = sheetNames.size();
            uint32 ver = T::getVersion ();
            ofile.serial (nbEntries);
            ofile.serial (ver);
            ofile.serialCont(container);
            ofile.close ();
        }
    }
    catch (NLMISC::Exception &e)
    {
        nlinfo ("loadForm(): Exception during saving the packed file, it will be recreated next launch (%s)", e.what());
    }

    // housekeeping
    sheetNames.clear ();
}

// ***************************************************************************
template <class T>
void loadForm (const std::string &sheetFilter, const std::string &packedFilename, std::map<std::string, T> &container, bool updatePackedSheet=true, bool errorIfPackedSheetNotGood=true)
{
    std::vector<std::string> vs;
    vs.push_back(sheetFilter);
    loadForm(vs, packedFilename, container, updatePackedSheet, errorIfPackedSheetNotGood);
}

// ***************************************************************************
template <class T>
void loadFormNoPackedSheet (const std::string &sheetFilter, std::map<NLMISC::CSheetId, T> &container, const std::string &wildcardFilter)
{
    std::vector<std::string> vs;
    vs.push_back(sheetFilter);
    loadFormNoPackedSheet(vs, container, wildcardFilter);
}

// ***************************************************************************
// variant with smart pointers, maintain with function above
template <class T>
void loadFormNoPackedSheet2 (const std::string &sheetFilter, std::map<NLMISC::CSheetId, T> &container, const std::string &wildcardFilter)
{
    std::vector<std::string> vs;
    vs.push_back(sheetFilter);
    loadFormNoPackedSheet2(vs, container, wildcardFilter);
}

// ***************************************************************************
template <class T>
void loadFormNoPackedSheet (const std::vector<std::string> &sheetFilters, std::map<NLMISC::CSheetId, T> &container, const std::string &wildcardFilter)
{
    // make sure the CSheetId singleton has been properly initialised
    NLMISC::CSheetId::init(false);

    // build a vector of the sheetFilters sheet ids (ie: "item")
    std::vector<NLMISC::CSheetId> sheetIds;
    std::vector<std::string> filenames;
    for (uint i = 0; i < sheetFilters.size(); i++)
        NLMISC::CSheetId::buildIdVector(sheetIds, filenames, sheetFilters[i]);


    // if there s no file, nothing to do
    if (sheetIds.empty())
        return;


    // compute sheets that needs to be recomputed
    std::vector<uint> NeededToRecompute;
    for (uint k = 0; k < filenames.size(); k++)
    {
        std::string p = NLMISC::CPath::lookup (filenames[k], false, false);
        if (p.empty()) continue;
        // check if wildcardok
        if(!wildcardFilter.empty() && !NLMISC::testWildCard(p,wildcardFilter)) continue;

        NeededToRecompute.push_back(k);
    }
    nlinfo ("%d sheets checked, %d need to be recomputed", filenames.size(), NeededToRecompute.size());


    NLMISC::TTime last = NLMISC::CTime::getLocalTime ();
    NLMISC::TTime start = NLMISC::CTime::getLocalTime ();
    NLGEORGES::UFormLoader *formLoader = NULL;
    NLMISC::CSmartPtr<NLGEORGES::UForm> form;
    std::vector<NLMISC::CSmartPtr<NLGEORGES::UForm> >   cacheFormList;

    // For all sheets need to recompute
    for (uint j = 0; j < NeededToRecompute.size(); j++)
    {
        if(NLMISC::CTime::getLocalTime () > last + 5000)
        {
            last = NLMISC::CTime::getLocalTime ();
            if(j>0)
                nlinfo ("%.0f%% completed (%d/%d), %d seconds remaining", (float)j*100.0/NeededToRecompute.size(),j,NeededToRecompute.size(), (NeededToRecompute.size()-j)*(last-start)/j/1000);
        }

        // create the georges loader if necessary
        if (formLoader == NULL)
        {
            NLMISC::WarningLog->addNegativeFilter("CFormLoader: Can't open the form file");
            formLoader = NLGEORGES::UFormLoader::createLoader ();
        }

        //  cache used to retain information (to optimize time).
        if (form)
            cacheFormList.push_back (form);

        // Load the form with given sheet id
        form = formLoader->loadForm (sheetIds[NeededToRecompute[j]].toString().c_str ());
        if (form)
        {
            // add the new creature, it could be already loaded by the packed sheets but will be overwritten with the new one
            typedef typename std::map<NLMISC::CSheetId, T>::iterator TType1;
            typedef typename std::pair<TType1, bool> TType2;
            TType2 res = container.insert(std::make_pair(sheetIds[NeededToRecompute[j]],T()));

            (*res.first).second.readGeorges (form, sheetIds[NeededToRecompute[j]]);
        }
    }

    if(NeededToRecompute.size() > 0)
        nlinfo ("%d seconds to recompute %d sheets", (uint32)(NLMISC::CTime::getLocalTime()-start)/1000, NeededToRecompute.size());

    // free the georges loader if necessary
    if (formLoader != NULL)
    {
        NLGEORGES::UFormLoader::releaseLoader (formLoader);
        NLMISC::WarningLog->removeFilter ("CFormLoader: Can't open the form file");
    }

    // housekeeping
    sheetIds.clear ();
    filenames.clear ();
}

// ***************************************************************************
// variant with smart pointers, maintain with function above
template <class T>
void loadFormNoPackedSheet2 (const std::vector<std::string> &sheetFilters, std::map<NLMISC::CSheetId, NLMISC::CSmartPtr<T> > &container, const std::string &wildcardFilter)
{
    // make sure the CSheetId singleton has been properly initialised
    NLMISC::CSheetId::init(false);

    // build a vector of the sheetFilters sheet ids (ie: "item")
    std::vector<NLMISC::CSheetId> sheetIds;
    std::vector<std::string> filenames;
    for (uint i = 0; i < sheetFilters.size(); i++)
        NLMISC::CSheetId::buildIdVector(sheetIds, filenames, sheetFilters[i]);


    // if there's no file, nothing to do
    if (sheetIds.empty())
        return;


    // compute sheets that needs to be recomputed
    std::vector<uint> NeededToRecompute;
    for (uint k = 0; k < filenames.size(); k++)
    {
        std::string p = NLMISC::CPath::lookup (filenames[k], false, false);
        if (p.empty()) continue;
        // check if wildcardok
        if(!wildcardFilter.empty() && !NLMISC::testWildCard(p,wildcardFilter)) continue;

        NeededToRecompute.push_back(k);
    }
    nlinfo ("%d sheets checked, %d need to be recomputed", filenames.size(), NeededToRecompute.size());


    NLMISC::TTime last = NLMISC::CTime::getLocalTime ();
    NLMISC::TTime start = NLMISC::CTime::getLocalTime ();
    NLGEORGES::UFormLoader *formLoader = NULL;
    NLMISC::CSmartPtr<NLGEORGES::UForm> form;
    std::vector<NLMISC::CSmartPtr<NLGEORGES::UForm> >   cacheFormList;

    // For all sheets need to recompute
    for (uint j = 0; j < NeededToRecompute.size(); j++)
    {
        if(NLMISC::CTime::getLocalTime () > last + 5000)
        {
            last = NLMISC::CTime::getLocalTime ();
            if(j>0)
                nlinfo ("%.0f%% completed (%d/%d), %d seconds remaining", (float)j*100.0/NeededToRecompute.size(),j,NeededToRecompute.size(), (NeededToRecompute.size()-j)*(last-start)/j/1000);
        }

        // create the georges loader if necessary
        if (formLoader == NULL)
        {
            NLMISC::WarningLog->addNegativeFilter("CFormLoader: Can't open the form file");
            formLoader = NLGEORGES::UFormLoader::createLoader ();
        }

        //  cache used to retain information (to optimize time).
        if (form)
            cacheFormList.push_back (form);

        // Load the form with given sheet id
        form = formLoader->loadForm (sheetIds[NeededToRecompute[j]].toString().c_str ());
        if (form)
        {
            // add the new creature, it could be already loaded by the packed sheets but will be overwritten with the new one
            typedef typename std::map<NLMISC::CSheetId, NLMISC::CSmartPtr<T> >::iterator TType1;
            typedef typename std::pair<TType1, bool> TType2;
            TType2 res = container.insert(std::make_pair(sheetIds[NeededToRecompute[j]], NLMISC::CSmartPtr<T>(new T())));

            (*res.first).second->readGeorges (form, sheetIds[NeededToRecompute[j]]);
        }
    }

    if(NeededToRecompute.size() > 0)
        nlinfo ("%d seconds to recompute %d sheets", (uint32)(NLMISC::CTime::getLocalTime()-start)/1000, NeededToRecompute.size());

    // free the georges loader if necessary
    if (formLoader != NULL)
    {
        NLGEORGES::UFormLoader::releaseLoader (formLoader);
        NLMISC::WarningLog->removeFilter ("CFormLoader: Can't open the form file");
    }

    // housekeeping
    sheetIds.clear ();
    filenames.clear ();
}

#endif // NL_LOAD_FORM_H

/* End of load_form.h */