From 8233c20a71d6b47f32261e03402cfcadab11c585 Mon Sep 17 00:00:00 2001 From: Ashley Aiken Date: Fri, 15 Jan 2021 16:01:03 -0500 Subject: [PATCH 1/7] initial addition of start and end dates from manifest --- koku/masu/external/kafka_msg_handler.py | 5 +++++ koku/masu/processor/tasks.py | 13 +++++++++---- koku/masu/util/ocp/common.py | 4 ++++ 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/koku/masu/external/kafka_msg_handler.py b/koku/masu/external/kafka_msg_handler.py index ea2acd744b..a6701044c6 100644 --- a/koku/masu/external/kafka_msg_handler.py +++ b/koku/masu/external/kafka_msg_handler.py @@ -495,6 +495,8 @@ def summarize_manifest(report_meta): provider_uuid = report_meta.get("provider_uuid") schema_name = report_meta.get("schema_name") provider_type = report_meta.get("provider_type") + start_date = report_meta.get("start") + end_date = report_meta.get("end") with ReportManifestDBAccessor() as manifest_accesor: if manifest_accesor.manifest_ready_for_summary(manifest_id): @@ -504,6 +506,9 @@ def summarize_manifest(report_meta): "provider_uuid": provider_uuid, "manifest_id": manifest_id, } + if start_date and end_date: + report_meta["start"] = start_date + report_meta["end"] = end_date async_id = summarize_reports.delay([report_meta]) return async_id diff --git a/koku/masu/processor/tasks.py b/koku/masu/processor/tasks.py index 1d500f56ee..587fc826b4 100644 --- a/koku/masu/processor/tasks.py +++ b/koku/masu/processor/tasks.py @@ -271,9 +271,15 @@ def summarize_reports(reports_to_summarize): # required. with ReportManifestDBAccessor() as manifest_accesor: if manifest_accesor.manifest_ready_for_summary(report.get("manifest_id")): - start_date = DateAccessor().today() - datetime.timedelta(days=2) - start_date = start_date.strftime("%Y-%m-%d") - end_date = DateAccessor().today().strftime("%Y-%m-%d") + if report.get("start") and report.get("end"): + LOG.info("using start and end dates from the manifest") + start_date = report.get("start") + end_date = report.get("end") + else: + LOG.info("generating start and end dates for manifest") + start_date = DateAccessor().today() - datetime.timedelta(days=2) + start_date = start_date.strftime("%Y-%m-%d") + end_date = DateAccessor().today().strftime("%Y-%m-%d") LOG.info("report to summarize: %s", str(report)) update_summary_tables.delay( report.get("schema_name"), @@ -314,7 +320,6 @@ def update_summary_tables(schema_name, provider, provider_uuid, start_date, end_ LOG.info(stmt) updater = ReportSummaryUpdater(schema_name, provider_uuid, manifest_id) - start_date, end_date = updater.update_daily_tables(start_date, end_date) updater.update_summary_tables(start_date, end_date) diff --git a/koku/masu/util/ocp/common.py b/koku/masu/util/ocp/common.py index 20589a21ef..d2b0a969d1 100644 --- a/koku/masu/util/ocp/common.py +++ b/koku/masu/util/ocp/common.py @@ -138,6 +138,10 @@ def get_report_details(report_directory): payload_dict = json.load(file) payload_dict["date"] = parser.parse(payload_dict["date"]) payload_dict["manifest_path"] = manifest_path + # parse start and end dates if in manifest + for field in ["start", "end"]: + if payload_dict.get(field): + payload_dict[field] = parser.parse(payload_dict[field]) except (OSError, IOError, KeyError) as exc: LOG.error("Unable to extract manifest data: %s", exc) From 9304d43dbd3d13cb4a0e8c0e4e40f2f3aa7f855b Mon Sep 17 00:00:00 2001 From: Ashley Aiken Date: Wed, 20 Jan 2021 16:18:39 -0500 Subject: [PATCH 2/7] minor changes to get the start/end date processing working --- Makefile | 7 +++++++ koku/masu/processor/tasks.py | 4 ++-- koku/masu/util/ocp/common.py | 4 +++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 5bbd31b61f..119ad09054 100644 --- a/Makefile +++ b/Makefile @@ -101,6 +101,7 @@ help: @echo " @param schema - (optional) schema name. Default: 'acct10001'." @echo " superuser create a Django super user" @echo " unittest run unittests" + @echo " local-upload-data upload data to Ingress if it is up and running locally" @echo "" @echo "--- Commands using Docker Compose ---" @echo " docker-up run docker-compose up --build -d" @@ -539,6 +540,12 @@ backup-local-db-dir: @cd - >/dev/null $(DOCKER_COMPOSE) start db +local-upload-data: + curl -vvvv -F "upload=@$(file);type=application/vnd.redhat.hccm.$(basename $(basename $(notdir $(file))))+tgz" \ + -H "x-rh-identity: eyJpZGVudGl0eSI6IHsiYWNjb3VudF9udW1iZXIiOiAiMTIzNDUiLCAiaW50ZXJuYWwiOiB7Im9yZ19pZCI6ICI1NDMyMSJ9fX0=" \ + -H "x-rh-request_id: testtesttest" \ + localhost:8080/api/ingress/v1/upload + restore-local-db-dir: @cd $(TOPDIR) diff --git a/koku/masu/processor/tasks.py b/koku/masu/processor/tasks.py index 587fc826b4..2c687deac7 100644 --- a/koku/masu/processor/tasks.py +++ b/koku/masu/processor/tasks.py @@ -273,8 +273,8 @@ def summarize_reports(reports_to_summarize): if manifest_accesor.manifest_ready_for_summary(report.get("manifest_id")): if report.get("start") and report.get("end"): LOG.info("using start and end dates from the manifest") - start_date = report.get("start") - end_date = report.get("end") + start_date = parser.parse(report.get("start")).strftime("%Y-%m-%d") + end_date = parser.parse(report.get("end")).strftime("%Y-%m-%d") else: LOG.info("generating start and end dates for manifest") start_date = DateAccessor().today() - datetime.timedelta(days=2) diff --git a/koku/masu/util/ocp/common.py b/koku/masu/util/ocp/common.py index d2b0a969d1..3ad1358730 100644 --- a/koku/masu/util/ocp/common.py +++ b/koku/masu/util/ocp/common.py @@ -127,7 +127,9 @@ def get_report_details(report_directory): payload_date: DateTime, manifest_path: String, uuid: String, - manifest_path: String" + manifest_path: String", + start: DateTime, + end: DateTime """ manifest_path = "{}/{}".format(report_directory, "manifest.json") From 05ca6755c0379f0caab0dc2e2c67886ab0584b15 Mon Sep 17 00:00:00 2001 From: Ashley Aiken Date: Wed, 20 Jan 2021 16:42:00 -0500 Subject: [PATCH 3/7] fix lint errors --- koku/masu/processor/tasks.py | 2 +- koku/masu/util/ocp/common.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/koku/masu/processor/tasks.py b/koku/masu/processor/tasks.py index 2c687deac7..df8fbff158 100644 --- a/koku/masu/processor/tasks.py +++ b/koku/masu/processor/tasks.py @@ -275,7 +275,7 @@ def summarize_reports(reports_to_summarize): LOG.info("using start and end dates from the manifest") start_date = parser.parse(report.get("start")).strftime("%Y-%m-%d") end_date = parser.parse(report.get("end")).strftime("%Y-%m-%d") - else: + else: LOG.info("generating start and end dates for manifest") start_date = DateAccessor().today() - datetime.timedelta(days=2) start_date = start_date.strftime("%Y-%m-%d") diff --git a/koku/masu/util/ocp/common.py b/koku/masu/util/ocp/common.py index 3ad1358730..987b3263d8 100644 --- a/koku/masu/util/ocp/common.py +++ b/koku/masu/util/ocp/common.py @@ -141,8 +141,8 @@ def get_report_details(report_directory): payload_dict["date"] = parser.parse(payload_dict["date"]) payload_dict["manifest_path"] = manifest_path # parse start and end dates if in manifest - for field in ["start", "end"]: - if payload_dict.get(field): + for field in ["start", "end"]: + if payload_dict.get(field): payload_dict[field] = parser.parse(payload_dict[field]) except (OSError, IOError, KeyError) as exc: LOG.error("Unable to extract manifest data: %s", exc) From a1773ed4911fa186b3bdc2429175e9ec52989003 Mon Sep 17 00:00:00 2001 From: Ashley Aiken Date: Thu, 21 Jan 2021 14:32:46 -0500 Subject: [PATCH 4/7] initial unit test additions --- koku/masu/test/data/ocp/payload2.tar.gz | Bin 0 -> 1484 bytes .../test/external/test_kafka_msg_handler.py | 29 ++++++++++++++++++ koku/masu/test/processor/test_tasks.py | 14 ++++++++- 3 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 koku/masu/test/data/ocp/payload2.tar.gz diff --git a/koku/masu/test/data/ocp/payload2.tar.gz b/koku/masu/test/data/ocp/payload2.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..885415ff1c3218c73faf33a238929a89d89bc5bd GIT binary patch literal 1484 zcma*l`#aNp9LMoRWxIvwbk}ajG`EULi@N2w9Ol?$D5tIrsjY^sZtk18snCkC8#zs+ znK@<6;daO^a>(ftOE#NoiRMrdE_2xZ-TVXhuaD33{rc&nO#y=p7~snw#kqi?$a1sY z{ZXM3HLceD8Yu9yMP5)tYe~ZLJuVn?>wI?!m2U|S zXa9EMB9~L^iy=YBlZM0d*~yZq{oul+GfM^u=Yx`UC{GAlC3;O^chGv|bph02dZXXZ z0i4oK;8-{a(oKY#D`v(1S?EqZcgHseYrciu!KRV0nSPf2#5}ij( zoJtjh^ndO+)PxrIO>R2iu&z8E%Yl`t+)%bEANQZl!&oTo|PfsH8*5DOqLt5c$ z<4mct6{=wlW))UjiMqL%+?4FwK7-q}l1ilfp>3m=gAl(}Kw=-S%* zqyk-v$28H5iMp_>?n8GUJx*~va3vwsARrl%`%;GVVE}v1%g|MZ-Xt9uVV(;hQR!Qk$MT1t^Jcs%_7R_oY)p?z`(f5k$kTTTDYakA z@1y&DuX08m{xq9q9_WKq?Qb0|4ySGOxXz$ zV+gO&R@Z#XR$^bBb%^dWuh&HviV{qWtSeAsOg&+eEsI;hWoAE=&zC+tz~F>7eL2mFL(BwJGC<7Q=Ss|Ld3=a$W;%i{m0!)L9p3aNp&ULW!3uYoVnH8;0)z%f#FowDD5C-Ek&F63PE=-HY^~( zGvE>34E-zq1@Uf9Rd5fOSHL~&weCBtE~=|a&Uw<$8D+YASetgHr?7V104K&hS7wZI z&j=NQ3DeCvWLhLXrhfqm9Nopz4fa&fw>q@ElR)I1`BryC-WA-DkEd;otINl=wvKF_ zK1Z9jiM8dOAW`1k-s)0Auz@J656!kAM->O9nWBnE6h_P1kXJS4qy-HL)BCi(NDhws za}#Nk{0siLDsyI#OzPPDM-v4dq@c Date: Thu, 21 Jan 2021 14:36:41 -0500 Subject: [PATCH 5/7] Lint again why isn't precommit fixing this before i push ha --- koku/masu/test/external/test_kafka_msg_handler.py | 4 +++- koku/masu/test/processor/test_tasks.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/koku/masu/test/external/test_kafka_msg_handler.py b/koku/masu/test/external/test_kafka_msg_handler.py index 4e5b3d2de4..818e96260c 100644 --- a/koku/masu/test/external/test_kafka_msg_handler.py +++ b/koku/masu/test/external/test_kafka_msg_handler.py @@ -485,7 +485,9 @@ def test_extract_payload_dates(self): with patch("masu.external.kafka_msg_handler.record_report_status", returns=None): msg_handler.extract_payload(payload_url, "test_request_id") expected_path = "{}/{}/{}/".format( - Config.INSIGHTS_LOCAL_REPORT_DIR, "5997a261-f23e-45d1-8e01-ee3c765f3aec", "20210101-20210201" + Config.INSIGHTS_LOCAL_REPORT_DIR, + "5997a261-f23e-45d1-8e01-ee3c765f3aec", + "20210101-20210201", ) self.assertTrue(os.path.isdir(expected_path)) shutil.rmtree(fake_dir) diff --git a/koku/masu/test/processor/test_tasks.py b/koku/masu/test/processor/test_tasks.py index cbf827b3e3..724597c1bc 100644 --- a/koku/masu/test/processor/test_tasks.py +++ b/koku/masu/test/processor/test_tasks.py @@ -371,7 +371,7 @@ def test_summarize_reports_processing_list(self, mock_update_summary): report_meta["provider_uuid"] = self.ocp_test_provider_uuid report_meta["manifest_id"] = 1 - # add a report with start/end dates specified + # add a report with start/end dates specified report2_meta = {} report2_meta["start_date"] = str(DateHelper().today) report2_meta["schema_name"] = self.schema From 2b649a98ebe6e54c906a366247cea584ef0b6958 Mon Sep 17 00:00:00 2001 From: Ashley Aiken Date: Thu, 21 Jan 2021 14:43:34 -0500 Subject: [PATCH 6/7] remove unused import --- koku/masu/test/processor/test_tasks.py | 1 - 1 file changed, 1 deletion(-) diff --git a/koku/masu/test/processor/test_tasks.py b/koku/masu/test/processor/test_tasks.py index 724597c1bc..587bfd368b 100644 --- a/koku/masu/test/processor/test_tasks.py +++ b/koku/masu/test/processor/test_tasks.py @@ -30,7 +30,6 @@ from uuid import uuid4 import faker -from dateutil import parser from dateutil import relativedelta from django.core.cache import caches from django.db.models import Max From 78206153081b1e4e8b6c7f1b38a476691202f1f2 Mon Sep 17 00:00:00 2001 From: Ashley Aiken Date: Thu, 21 Jan 2021 15:00:58 -0500 Subject: [PATCH 7/7] add another test --- .../test/external/test_kafka_msg_handler.py | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/koku/masu/test/external/test_kafka_msg_handler.py b/koku/masu/test/external/test_kafka_msg_handler.py index 818e96260c..8156cb93b4 100644 --- a/koku/masu/test/external/test_kafka_msg_handler.py +++ b/koku/masu/test/external/test_kafka_msg_handler.py @@ -441,6 +441,56 @@ def manifest_ready_for_summary(self, manifest_id): msg_handler.summarize_manifest(report_meta) mock_summarize_reports.assert_not_called() + def test_summarize_manifest_dates(self): + """Test report summarization.""" + report_meta = { + "schema_name": "test_schema", + "manifest_id": "1", + "provider_uuid": uuid.uuid4(), + "provider_type": "OCP", + "compression": "UNCOMPRESSED", + "file": "/path/to/file.csv", + "start": str(datetime.today()), + "end": str(datetime.today()), + } + expected_meta = { + "schema_name": report_meta.get("schema_name"), + "provider_type": report_meta.get("provider_type"), + "provider_uuid": report_meta.get("provider_uuid"), + "manifest_id": report_meta.get("manifest_id"), + "start": report_meta.get("start"), + "end": report_meta.get("end"), + } + + class FakeManifest: + def __init__(self, num_processed_files=1, num_total_files=1): + self.num_processed_files = num_processed_files + self.num_total_files = num_total_files + + def get_manifest_by_id(self, manifest_id): + return self + + def manifest_ready_for_summary(self, manifest_id): + return self.num_processed_files == self.num_total_files + + # Check when manifest is done + mock_manifest_accessor = FakeManifest(num_processed_files=2, num_total_files=2) + + with patch("masu.external.kafka_msg_handler.ReportManifestDBAccessor") as mock_accessor: + mock_accessor.return_value.__enter__.return_value = mock_manifest_accessor + with patch("masu.external.kafka_msg_handler.summarize_reports.delay") as mock_summarize_reports: + msg_handler.summarize_manifest(report_meta) + mock_summarize_reports.assert_called_with([expected_meta]) + + # Check when manifest is not done + mock_manifest_accessor = FakeManifest(num_processed_files=1, num_total_files=2) + + with patch("masu.external.kafka_msg_handler.ReportManifestDBAccessor") as mock_accessor: + mock_accessor.return_value.__enter__.return_value = mock_manifest_accessor + with patch("masu.external.kafka_msg_handler.summarize_reports.delay") as mock_summarize_reports: + msg_handler.summarize_manifest(report_meta) + mock_summarize_reports.assert_not_called() + def test_extract_payload(self): """Test to verify extracting payload is successful."""