Add delete option to deploy script

If this option enabled after some time all other files (that wasn't
updated) will be removed from bucket. Wait time by default is 2 minutes
(html cache time), so removing old objects should not cause any issues
with consistency.

ci/deploy.py

@@ -4,6 +4,7 @@ import os
 import boto3
 import mimetypes
 import argparse
+import time
 
 
 UPLOAD_ORDER = [
@@ -17,6 +18,7 @@ CACHE_RULES = [
 ]
 
 DEFAULT_PRIORITY = 0
+REMOVE_TIMEOUT = 60 * 2
 
 
 def deployment_files(deploy_dir):
@@ -59,21 +61,45 @@ def file_options(file_name):
     return extra_config
 
 
-def update_site(deploy_dir, bucket_name):
+def remove_old(bucket, new_files):
+    new_files_set = frozenset(new_files)
+    removed_files = []
+
+    for s3_object in bucket.objects.all():
+        if s3_object.key not in new_files_set:
+            removed_files.append(s3_object.key)
+            print("Marked for remove: {}".format(s3_object.key))
+
+    # TODO: check errors
+    bucket.delete_objects(Delete={
+        'Objects': [{'Key': key} for key in removed_files],
+        'Quiet': True
+    })
+    print("Objects removed")
+
+
+def update_site(deploy_dir, bucket_name, delete=False):
     s3 = boto3.resource('s3')
     bucket = s3.Bucket(bucket_name)
+    updated_files = sorted_files(deploy_dir)
 
-    for file_name in sorted_files(deploy_dir):
+    for file_name in updated_files:
         file_path = os.path.join(deploy_dir, file_name)
         extra = file_options(file_name)
         print("Uploading: {}".format(file_name))
         bucket.upload_file(file_path, file_name, extra)
 
+    if delete:
+        print("Wait for cache invalidation")  # till cache become old
+        time.sleep(REMOVE_TIMEOUT)
+        remove_old(bucket, updated_files)
+
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Deploy site to s3')
+    parser.add_argument('--delete', action='store_true', help="Remove old files")
     parser.add_argument("deploy_dir")
     parser.add_argument("s3_bucket")
     args = parser.parse_args()
-    update_site(args.deploy_dir, args.s3_bucket)
+    update_site(args.deploy_dir, args.s3_bucket, delete=args.delete)
     print("Done")