From 70ed5feb9d49c7bc655aa0d0ae338595c51ebe91 Mon Sep 17 00:00:00 2001
From: Sandeep Sukhani <sandeep.d.sukhani@gmail.com>
Date: Tue, 10 Sep 2019 16:06:27 +0530
Subject: [PATCH] Change unit of duration params to hours to align it with
duration config at other places in Loki (#998)
* Change unit of duration params to hours to align it with duration config at other places in Loki
other configurations like periodic table duration, retention are set in hours so this change makes
duration config for backup tool consistent and easy to manage configurations
* update parent docker image for bigtable backup tool
---
tools/bigtable-backup/Dockerfile | 2 +-
tools/bigtable-backup/bigtable-backup.py | 22 +++++++++-------------
2 files changed, 10 insertions(+), 14 deletions(-)
diff --git a/tools/bigtable-backup/Dockerfile b/tools/bigtable-backup/Dockerfile
index c278fb4a..bfee013a 100644
--- a/tools/bigtable-backup/Dockerfile
+++ b/tools/bigtable-backup/Dockerfile
@@ -1,4 +1,4 @@
-FROM grafana/bigtable-backup:master-63aad9c
+FROM grafana/bigtable-backup:master-c32bd7b
RUN apk add --update --no-cache python3 python3-dev git \
&& pip3 install --no-cache-dir --upgrade pip
COPY bigtable-backup.py bigtable-backup.py
diff --git a/tools/bigtable-backup/bigtable-backup.py b/tools/bigtable-backup/bigtable-backup.py
index a45be0c4..cf122ac4 100644
--- a/tools/bigtable-backup/bigtable-backup.py
+++ b/tools/bigtable-backup/bigtable-backup.py
@@ -42,7 +42,7 @@ def ensure_backups(args):
backups = list_backups(args.destination_path)
if args.period_from == None:
- period_from = datetime.utcnow() - timedelta(days=args.duration)
+ period_from = datetime.utcnow() - timedelta(seconds=args.duration)
args.period_from = valid_date(period_from.strftime("%Y-%m-%d"))
args.period_to = valid_date(datetime.utcnow().strftime("%Y-%m-%d"))
@@ -104,11 +104,10 @@ def set_ensure_backups_specific_metrics(args, num_backups_deleted, active_table_
duration = args.duration
if args.duration == None:
- duration = (args.period_to - args.period_from).days
+ duration = (args.period_to - args.period_from).seconds
# there should be 1 backup per inactive table
- periodic_table_duration_in_days = args.periodic_table_duration / 86400
- bigtable_backup_job_expected_inactive_table_backups.set(int(duration/periodic_table_duration_in_days))
+ bigtable_backup_job_expected_inactive_table_backups.set(int(duration/args.periodic_table_duration))
bigtable_backup_job_backups_deleted.set(num_backups_deleted)
@@ -140,9 +139,9 @@ def valid_date(s):
raise argparse.ArgumentTypeError(msg)
-def valid_periodic_duration(s):
+def valid_duration(s):
try:
- return int(s) * 86400
+ return int(s) * 3600
except ValueError:
msg = "Not a valid duration: '{0}'.".format(s)
raise argparse.ArgumentTypeError(msg)
@@ -152,9 +151,6 @@ def valid_table_id_prefix(s):
if not str(s).endswith("_"):
return str(s) + "_"
-def valid_int(s):
- return int(s)
-
def create_backup(table_id, args):
popen = subprocess.Popen(['bigtable-backup', 'create', '--bigtable-table-id-prefix', table_id,
@@ -225,8 +221,8 @@ def main():
help="GCS path where data should be written. For example, gs://mybucket/somefolder/")
parser.add_argument("--temp-prefix", required=True,
help="Path and filename prefix for writing temporary files. ex: gs://MyBucket/tmp")
- parser.add_argument("--periodic-table-duration", required=True, type=valid_periodic_duration,
- help="Periodic config set for loki tables in days")
+ parser.add_argument("--periodic-table-duration", required=True, type=valid_duration,
+ help="Periodic config set for loki tables in hours")
parser.add_argument("--prom-push-gateway-endpoint", default="localhost:9091", help="Endpoint where metrics are to be pushed")
parser.add_argument("--namespace", default="default", help="namespace while reporting metrics")
@@ -236,8 +232,8 @@ def main():
ensure_backups_parser = subparser.add_parser(job_ensure_backups,
help="Ensure backups of right tables exist")
- ensure_backups_parser.add_argument('--duration', help="Duration in previous consecutive days for which backups should exist. "
- "Must not be set with --period-from and --period-to", type=valid_int)
+ ensure_backups_parser.add_argument('--duration', help="Duration in hours for which backups should exist. "
+ "Must not be set with --period-from and --period-to", type=valid_duration)
ensure_backups_parser.add_argument('--period-from', type=valid_date, help="Backups should exist starting from the date. Must not be set with --duration")
ensure_backups_parser.add_argument('--period-to', type=valid_date,
default=datetime.utcnow().strftime("%Y-%m-%d"))
--
GitLab