Enhance scraper log archiving

This commit implements putting the log archive files in the
logarchive subdirectory to prevent clutter.

It also implements deleting old log files per the following
configuration file (or command line) parameters:

deleteoldlogarchives=0|1 (default true)
logarchiveretainnumfiles=x (default 14)

deleteoldlogarchives defaults to true, which then means that
every time the log archiver is triggered (typically at midnight UTC)
it will archive the current scraper log file into the logarchive
directory, and then apply the rule to retain the x most recent files,
where x is specified by logarchiveretainnumfiles.

An immediate archive (and directory cleanup) can be triggered by issuing
the rpc command archivescraperlog.

If deletelogarchives is set to false, then the scraper.log files are
archived into the logarchive directory, but never deleted.

Author: jamescowens

src/scraper/scraper.cpp

@@ -208,6 +208,23 @@ class logger
 
         if (fDebug) LogPrintf("INFO: Scraper: Logger: ArchiveCheckDate %s, PrevArchiveCheckDate %s", ssArchiveCheckDate.str(), ssPrevArchiveCheckDate.str());
 
+        fs::path LogArchiveDir = pathDataDir / "logarchive";
+
+        // Check to see if the log archive directory exists and is a directory. If not create it.
+        if (fs::exists(LogArchiveDir))
+        {
+            // If it is a normal file, this is not right. Remove the file and replace with the log archive directory.
+            if (fs::is_regular_file(LogArchiveDir))
+            {
+                fs::remove(LogArchiveDir);
+                fs::create_directory(LogArchiveDir);
+            }
+        }
+        else
+        {
+            fs::create_directory(LogArchiveDir);
+        }
+
         if (fImmediate || (fArchiveDaily && ArchiveCheckDate > PrevArchiveCheckDate))
         {
             {
@@ -221,7 +238,7 @@ class logger
 
                 plogfile = pathDataDir / "scraper.log";
                 pfile_temp = pathDataDir / ("scraper-" + DateTimeStrFormat("%Y%m%d%H%M%S", nTime) + ".log");
-                pfile_out = pathDataDir / ("scraper-" + DateTimeStrFormat("%Y%m%d%H%M%S", nTime) + ".log.gz");
+                pfile_out = LogArchiveDir / ("scraper-" + DateTimeStrFormat("%Y%m%d%H%M%S", nTime) + ".log.gz");
 
                 try
                 {
@@ -264,6 +281,40 @@ class logger
 
             fs::remove(pfile_temp);
 
+            bool fDeleteOldLogArchives = GetBoolArg("-deleteoldlogarchives", true);
+
+            if (fDeleteOldLogArchives)
+            {
+                unsigned int nRetention = (unsigned int)GetArg("-logarchiveretainnumfiles", 14);
+
+                std::set<fs::directory_entry, std::greater <fs::directory_entry>> SortedDirEntries;
+
+                // Iterate through the log archive directory and delete the oldest files beyond the retention rule
+                // The names are in format scraper-YYYYMMDDHHMMSS for the scraper logs, so filter by containing scraper
+                // The greater than sort in the set should then return descending order by datetime.
+                for (fs::directory_entry& DirEntry : fs::directory_iterator(LogArchiveDir))
+                {
+                    std::string sFilename = DirEntry.path().filename().string();
+                    size_t FoundPos = sFilename.find("scraper");
+
+                    if (FoundPos != string::npos) SortedDirEntries.insert(DirEntry);
+                }
+
+                // Now iterate through set of filtered filenames. Delete all files greater than retention count.
+                unsigned int i = 0;
+                for (auto const& iter : SortedDirEntries)
+                {
+                    if (i >= nRetention)
+                    {
+                        fs::remove(iter.path());
+
+                        LogPrintf("INFO: logger: Removed old archive gzip file %s.", iter.path().filename().string());
+                    }
+
+                    ++i;
+                }
+            }
+
             return true;
         }
         else
@@ -718,7 +769,7 @@ void Scraper(bool bSingleShot)
                 fs::path plogfile_out;
 
                 if (log.archive(false, plogfile_out))
-                    _log(logattribute::INFO, "Scraper", "Archived scraper.log to " + plogfile_out.string());
+                    _log(logattribute::INFO, "Scraper", "Archived scraper.log to " + plogfile_out.filename().string());
 
 
                 sbage = SuperblockAge();
@@ -4184,11 +4235,6 @@ UniValue archivescraperlog(const UniValue& params, bool fHelp)
     fs::path pfile_out;
     bool ret = log.archive(true, pfile_out);
 
-    if(!ret)
-        return UniValue(ret);
-    else
-        return UniValue(pfile_out.c_str());
-
     return UniValue(ret);
 }