Use log folder for activity log backups

activitylog/management/commands/delete_old_activitylogs.py

@@ -1,5 +1,6 @@
 import csv
 import os
+from pathlib import Path
 import subprocess
 
 from dateutil.relativedelta import relativedelta
@@ -31,14 +32,16 @@ def handle(self, *args, **options):
         # set cutoff to beginning of this day <age> years ago
         cutoff = (now-relativedelta(years=age)).replace(hour=0, minute=0, second=0, microsecond=0)
         filename = f"{settings.S3_LOG_BACKUP_ROOT_FILENAME}_{cutoff.strftime('%Y-%m-%d')}_{now.strftime('%Y%m%d%H%M%S')}.csv"
+        local_filepath = Path(settings.LOG_FOLDER) / "activitylogs_backup" / filename
+        local_filepath.parent.mkdir(exist_ok=True)
         s3_upload_path = os.path.join(settings.S3_LOG_BACKUP_PATH, filename)
         # Delete the empty logs first
         management.call_command('delete_empty_job_logs', cutoff.strftime('%Y%m%d'))
 
         old_logs = ActivityLog.objects.filter(timestamp__lt=cutoff)
         old_logs_count = old_logs.count()
         if old_logs_count > 0:
-            with open(filename, "w") as outfile:
+            with open(local_filepath, "w") as outfile:
                 wr = csv.writer(outfile)
                 wr.writerow([
                     smart_str(u"Timestamp"),
@@ -51,7 +54,7 @@ def handle(self, *args, **options):
                     ])
 
             subprocess.run(["aws", "s3", "cp", filename, s3_upload_path], check=True)
-            os.unlink(filename)
+            os.unlink(local_filepath)
 
             old_logs.delete()
             message = f"{old_logs_count} activitylogs older than {cutoff.strftime('%Y-%m-%d')} backed up to s3 and deleted"

activitylog/tests.py

@@ -1,6 +1,7 @@
 import os
 import sys
 from io import StringIO
+from tempfile import TemporaryDirectory
 from unittest.mock import patch
 
 from datetime import datetime, timedelta
@@ -12,7 +13,7 @@
 from django.conf import settings
 from django.contrib.admin.sites import AdminSite
 from django.core import management
-from django.test import TestCase
+from django.test import TestCase, override_settings
 from django.utils import timezone
 
 from activitylog import admin
@@ -178,38 +179,43 @@ def setUp(self):
     def test_delete_default_old_logs(self, mock_now, mock_run):
         mock_now.return_value = self.mock_now
         self.assertEqual(ActivityLog.objects.count(), 3)
-        # no age, defaults to 1 yr
-        management.call_command('delete_old_activitylogs')
-        # 2 logs left - the one that's < 1 yrs old plus the new one to log this activity
-        self.assertEqual(ActivityLog.objects.count(), 2)
-        all_log_ids = ActivityLog.objects.values_list("id", flat=True)
-        for log in [self.log_25monthsold, self.log_37monthsold]:
-            self.assertNotIn(log.id, all_log_ids)
-        self.assertIn(self.log_11monthsold.id, all_log_ids)
-
-        self.assertEqual(mock_run.call_count, 1)
-        cutoff = (self.mock_now-relativedelta(years=1)).strftime('%Y-%m-%d')
-        filename = f"{settings.S3_LOG_BACKUP_ROOT_FILENAME}_{cutoff}_{self.mock_now.strftime('%Y%m%d%H%M%S')}.csv"
-        mock_run.assert_called_once_with(
-            ['aws', 's3', 'cp', filename, os.path.join(settings.S3_LOG_BACKUP_PATH, filename)], check=True
-        )
+        with TemporaryDirectory() as tmpdir:
+            with override_settings(LOG_FOLDER=tmpdir):
+                # no age, defaults to 1 yr
+                management.call_command('delete_old_activitylogs')
+                # 2 logs left - the one that's < 1 yrs old plus the new one to log this activity
+                self.assertEqual(ActivityLog.objects.count(), 2)
+                all_log_ids = ActivityLog.objects.values_list("id", flat=True)
+                for log in [self.log_25monthsold, self.log_37monthsold]:
+                    self.assertNotIn(log.id, all_log_ids)
+                self.assertIn(self.log_11monthsold.id, all_log_ids)
+
+                self.assertEqual(mock_run.call_count, 1)
+                cutoff = (self.mock_now-relativedelta(years=1)).strftime('%Y-%m-%d')
+                filename = f"{settings.S3_LOG_BACKUP_ROOT_FILENAME}_{cutoff}_{self.mock_now.strftime('%Y%m%d%H%M%S')}.csv"
+
+                mock_run.assert_called_once_with(
+                    ['aws', 's3', 'cp', filename, os.path.join(settings.S3_LOG_BACKUP_PATH, filename)], check=True
+                )
 
     @patch('activitylog.management.commands.delete_old_activitylogs.subprocess.run')
     @patch('activitylog.management.commands.delete_old_activitylogs.timezone.now')
     def test_delete_old_logs_with_args(self, mock_now, mock_run):
         mock_now.return_value = self.mock_now
         self.assertEqual(ActivityLog.objects.count(), 3)
-        management.call_command('delete_old_activitylogs', age=3)
-        # 3 logs left - the 2 that are < 3 yrs old plus the new one to log this activity
-        self.assertEqual(ActivityLog.objects.count(), 3)
-        all_log_ids = ActivityLog.objects.values_list("id", flat=True)
-        for log in [self.log_11monthsold, self.log_25monthsold]:
-            self.assertIn(log.id, all_log_ids)
-        self.assertNotIn(self.log_37monthsold.id, all_log_ids)
-
-        self.assertEqual(mock_run.call_count, 1)
-        cutoff = (self.mock_now-relativedelta(years=3)).strftime('%Y-%m-%d')
-        filename = f"{settings.S3_LOG_BACKUP_ROOT_FILENAME}_{cutoff}_{self.mock_now.strftime('%Y%m%d%H%M%S')}.csv"
-        mock_run.assert_called_once_with(
-            ['aws', 's3', 'cp', filename, os.path.join(settings.S3_LOG_BACKUP_PATH, filename)], check=True
-        )
+        with TemporaryDirectory() as tmpdir:
+            with override_settings(LOG_FOLDER=tmpdir):
+                management.call_command('delete_old_activitylogs', age=3)
+                # 3 logs left - the 2 that are < 3 yrs old plus the new one to log this activity
+                self.assertEqual(ActivityLog.objects.count(), 3)
+                all_log_ids = ActivityLog.objects.values_list("id", flat=True)
+                for log in [self.log_11monthsold, self.log_25monthsold]:
+                    self.assertIn(log.id, all_log_ids)
+                self.assertNotIn(self.log_37monthsold.id, all_log_ids)
+
+                self.assertEqual(mock_run.call_count, 1)
+                cutoff = (self.mock_now-relativedelta(years=3)).strftime('%Y-%m-%d')
+                filename = f"{settings.S3_LOG_BACKUP_ROOT_FILENAME}_{cutoff}_{self.mock_now.strftime('%Y%m%d%H%M%S')}.csv"
+                mock_run.assert_called_once_with(
+                    ['aws', 's3', 'cp', filename, os.path.join(settings.S3_LOG_BACKUP_PATH, filename)], check=True
+                )

pipsevents/settings.py

@@ -36,7 +36,8 @@
                   SHOW_VAT=(bool, True),
                   TESTING=(bool, False),
                   PAYMENT_METHOD=(str, "stripe"),
-                  ENFORCE_AUTO_CANCELLATION=(bool, False)       
+                  ENFORCE_AUTO_CANCELLATION=(bool, False),  
+                  LOG_FOLDER=(str, "logs") 
                   )
 
 environ.Env.read_env(root('pipsevents/.env'))  # reading .env file
@@ -46,6 +47,7 @@
 if not TESTING:  # pragma: no cover
     TESTING = any([test_str in arg for arg in sys.argv for test_str in ["test", "pytest"]])
 
+LOG_FOLDER = env("LOG_FOLDER")
 BASE_DIR = root()
 #
 # Quick-start development settings - unsuitable for production
@@ -275,7 +277,6 @@
 # #####LOGGING######
 
 if not TESTING and not LOCAL:  # pragma: no cover
-    LOG_FOLDER = env('LOG_FOLDER')
     LOG_FILE = os.path.join(LOG_FOLDER, 'pipsevents.log')
     log_file_permissions(LOG_FILE)
     LOGGING = {