Added Vue Dashboard

webis-de · janheinrichmerker · Nov 3, 2023 · Nov 3, 2023 · Nov 5, 2023 · Nov 5, 2023
commit 6eeeae90b7bc48b56f1b080a09c9b4062ff9c1be
diff --git a/archive_query_log/dashboard/.browserslistrc b/archive_query_log/dashboard/.browserslistrc
@@ -0,0 +1,4 @@
+> 1%
+last 2 versions
+not dead
+not ie 11
diff --git a/archive_query_log/dashboard/.editorconfig b/archive_query_log/dashboard/.editorconfig
@@ -0,0 +1,5 @@
+[*.{js,jsx,ts,tsx,vue}]
+indent_style = space
+indent_size = 2
+trim_trailing_whitespace = true
+insert_final_newline = true
diff --git a/archive_query_log/dashboard/.gitignore b/archive_query_log/dashboard/.gitignore
@@ -0,0 +1,22 @@
+.DS_Store
+node_modules
+/dist
+
+# local env files
+.env.local
+.env.*.local
+
+# Log files
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+
+# Editor directories and files
+.idea
+.vscode
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?
diff --git a/archive_query_log/dashboard/README.md b/archive_query_log/dashboard/README.md
@@ -0,0 +1,17 @@
+# 📈 aql-monitoring
+
+Monitor and manage the crawling of the [Archive Query Log](https://github.com/webis-de/archive-query-log).
+
+## Installation
+
+```shell
+npm install
+```
+
+TODO: Add better instructions.
+
+## Usage
+
+For starting the website, navigate to the repo and do
+```npm run dev-vite```
+in your terminal
diff --git a/archive_query_log/dashboard/components.d.ts b/archive_query_log/dashboard/components.d.ts
@@ -0,0 +1,16 @@
+/* eslint-disable */
+/* prettier-ignore */
+// @ts-nocheck
+// Generated by unplugin-vue-components
+// Read more: https://github.com/vuejs/core/pull/3399
+export {}
+
+declare module 'vue' {
+  export interface GlobalComponents {
+    Footer: typeof import('./src/components/Footer.vue')['default']
+    Header: typeof import('./src/components/Header.vue')['default']
+    Home: typeof import('./src/components/Home.vue')['default']
+    ProgressTable: typeof import('./src/components/ProgressTable.vue')['default']
+    StatisticsTable: typeof import('./src/components/StatisticsTable.vue')['default']
+  }
+}
diff --git a/archive_query_log/dashboard/index.html b/archive_query_log/dashboard/index.html
@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+    <meta charset="UTF-8" />
+    <title>Archive Query Log</title>
+</head>
+
+<body>
+    <div id="app"></div>
+    <script type="module" src="/src/main.ts"></script>
+</body>
+
+</html>
diff --git a/archive_query_log/legacy/__init__.py b/archive_query_log/legacy/__init__.py
 LOGGER: Logger = getLogger(__name__)

 if __name__ == "__main__":
    from archive_query_log.legacy.cli import main

    main()
diff --git a/archive_query_log/legacy/download/iterable.py b/archive_query_log/legacy/download/iterable.py
                record_url_header
            )
        except JSONDecodeError:
            LOGGER.warning(
                f"Could not index {record_url_header} from record "
                f"{record.rec_headers.get_header('WARC-Record-ID')}."
            )
            return None
        encoding = record.http_headers.get_header("Content-Type")
        if encoding is None:
            encoding = ""
        encoding = encoding.split(";")[-1].split("=")[-1].strip().lower()
        if encoding == "" or "/" in encoding:
            encoding = "utf8"
diff --git a/archive_query_log/legacy/model/__init__.py b/archive_query_log/legacy/model/__init__.py
        """
        MD5 hash of the original URL.
        """
        return md5(self.url.encode(), usedforsecurity=False).hexdigest()

    @cached_property
    def datetime(self) -> datetime:
diff --git a/archive_query_log/legacy/model/parse.py b/archive_query_log/legacy/model/parse.py
                ),
            )
        elif parser_type == "fragment_segment":
            from archive_query_log.legacy.queries.parse import \
                FragmentSegmentQueryParser
            return FragmentSegmentQueryParser(
                url_pattern=pattern(value["url_pattern"], IGNORECASE),
                ),
            )
        elif parser_type == "fragment_segment":
            from archive_query_log.legacy.queries.parse import \
                FragmentSegmentPageOffsetParser
            return FragmentSegmentPageOffsetParser(
                url_pattern=pattern(value["url_pattern"], IGNORECASE),
                ),
            )
        elif parser_type == "chatnoir":
            from archive_query_log.legacy.results.chatnoir import \
                ChatNoirResultsParser
            return ChatNoirResultsParser(
                url_pattern=pattern(value["url_pattern"], IGNORECASE),
diff --git a/archive_query_log/legacy/queries/parse.py b/archive_query_log/legacy/queries/parse.py
        if output_path.exists() and not self.overwrite:
            return
        output_path.parent.mkdir(parents=True, exist_ok=True)
        archived_urls: Iterable[ArchivedUrl] = ArchivedUrls(input_path)
        if self.verbose:
            # noinspection PyTypeChecker
            archived_urls = tqdm(
                desc="Parse SERP URLs",
                unit="URL",
            )
        archived_serp_urls_nullable = (
            self._parse_single(archived_url, focused)
            for archived_url in archived_urls
        )
            if archived_serp_url is not None
        )
        output_schema = ArchivedQueryUrl.schema()
        with (output_path.open("wb") as file,
              GzipFile(fileobj=file, mode="wb") as gzip_file,
              text_io_wrapper(gzip_file) as text_file):
            for archived_serp_url in archived_serp_urls:
            focused: bool,
    ) -> ArchivedQueryUrl | None:
        query: str | None = None
        for query_parser in self.query_parsers:
            query = query_parser.parse(archived_url)
            if query is not None:
                break

            return None

        page: int | None = None
        for page_parser in self.page_parsers:
            page = page_parser.parse(archived_url)
            if page is not None:
                break

            return None

        offset: int | None = None
        for offset_parser in self.offset_parsers:
            offset = offset_parser.parse(archived_url)
            if offset is not None:
                break

                ]

        if cdx_page is not None:
            if domain is None:
                raise RuntimeError(
                    "Domain must be specified when page is specified.")
            if len(domain_paths) < 1:
                raise RuntimeError(
                    "There must be exactly one domain path.")
            cdx_page_paths = [domain_paths[0] / f"{cdx_page:010}.jsonl.gz"]
        else:
            domain: str | None = None,
            cdx_page: int | None = None,
    ):
        pages_list: Sequence[_CdxPage] = self._service_pages(
            data_directory=data_directory,
            focused=focused,
            service=service,
            cdx_page=cdx_page,
        )

        if len(pages_list) == 0:
            return

        pages: Iterable[_CdxPage] = pages_list
        if len(pages_list) > 1:
            # noinspection PyTypeChecker
            pages = tqdm(
                pages,
diff --git a/archive_query_log/legacy/results/parse.py b/archive_query_log/legacy/results/parse.py
        if output_path.exists() and not self.overwrite:
            return
        output_path.parent.mkdir(parents=True, exist_ok=True)
        archived_serp_contents: Iterable[ArchivedRawSerp] = (
            ArchivedRawSerps(input_path))
        if self.verbose:
            # noinspection PyTypeChecker
                desc="Parse SERP WARC records",
                unit="record",
            )
        archived_parsed_serps_nullable = (
            self.parse_single(archived_serp_content)
            for archived_serp_content in archived_serp_contents
        )
                ]

        if cdx_page is not None:
            if domain is None:
                raise RuntimeError(
                    "Domain must be specified when page is specified.")
            if len(domain_paths) < 1:
                raise RuntimeError(
                    "There must be exactly one domain path.")
            cdx_page_paths = [domain_paths[0] / f"{cdx_page:010}"]
        else:
            domain: str | None = None,
            cdx_page: int | None = None,
    ):
        pages_list: Sequence[_CdxPage] = self._service_pages(
            data_directory=data_directory,
            focused=focused,
            service=service,
            cdx_page=cdx_page,
        )

        if len(pages_list) == 0:
            return

        pages: Iterable[_CdxPage] = pages_list
        if len(pages_list) > 1:
            # noinspection PyTypeChecker
            pages = tqdm(
                pages,
diff --git a/archive_query_log/legacy/services/__init__.py b/archive_query_log/legacy/services/__init__.py
            try:
                service = Service.schema(unknown="exclude").load(service_dict)
                services += [(service.name, service)]
            except ValidationError as e:
                if not ignore_parsing_errors:
                    raise ValueError(
                        f"Could not parse service {service_dict}") from e
        if not ignore_parsing_errors:
diff --git a/archive_query_log/legacy/urls/iterable.py b/archive_query_log/legacy/urls/iterable.py
        self._check_urls_path()

    def _check_urls_path(self):
        if not self.path.exists() or not self.path.is_file():
            raise ValueError(
                f"URLs path must be a file: {self.path}"
            )

    def __len__(self) -> int:
        with (self.path.open("rb") as file,
              GzipFile(fileobj=file, mode="rb") as gzip_file):
            return count_lines(gzip_file)

    def __iter__(self) -> Iterator[ArchivedUrl]:
        schema = ArchivedUrl.schema()
        with (self.path.open("rb") as file,
              GzipFile(fileobj=file, mode="rb") as gzip_file,
              text_io_wrapper(gzip_file) as text_file):
            for line in text_file:
                url = schema.loads(line)
                if isinstance(url, list):
                    raise ValueError(f"Expected one URL per line: {line}")
                yield url
diff --git a/archive_query_log/legacy/util/text.py b/archive_query_log/legacy/util/text.py


 def text_io_wrapper(file: IO[bytes] | IOBase) -> IO[str]:
    return TextIOWrapper(file)  # type: ignore
	LOGGER: Logger = getLogger(__name__)

	if __name__ == "__main__":
	from archive_query_log.legacy.cli import main
Check warning on line 12 in archive_query_log/legacy/__init__.py Codecov / codecov/patch archive_query_log/legacy/__init__.py#L12 `Added line #L12 was not covered by tests`

	main()
Check warning on line 14 in archive_query_log/legacy/__init__.py Codecov / codecov/patch archive_query_log/legacy/__init__.py#L14 `Added line #L14 was not covered by tests`
	record_url_header
	)
	except JSONDecodeError:
	LOGGER.warning(
Check warning on line 65 in archive_query_log/legacy/download/iterable.py Codecov / codecov/patch archive_query_log/legacy/download/iterable.py#L65 `Added line #L65 was not covered by tests`
	f"Could not index {record_url_header} from record "
	f"{record.rec_headers.get_header('WARC-Record-ID')}."
	)
	return None
	encoding = record.http_headers.get_header("Content-Type")
	if encoding is None:
	encoding = ""
Check warning on line 72 in archive_query_log/legacy/download/iterable.py Codecov / codecov/patch archive_query_log/legacy/download/iterable.py#L72 `Added line #L72 was not covered by tests`
	encoding = encoding.split(";")[-1].split("=")[-1].strip().lower()
	if encoding == "" or "/" in encoding:
	encoding = "utf8"
	"""
	MD5 hash of the original URL.
	"""
	return md5(self.url.encode(), usedforsecurity=False).hexdigest()
Check warning on line 64 in archive_query_log/legacy/model/__init__.py Codecov / codecov/patch archive_query_log/legacy/model/__init__.py#L64 `Added line #L64 was not covered by tests`

	@cached_property
	def datetime(self) -> datetime:
	),
	)
	elif parser_type == "fragment_segment":
	from archive_query_log.legacy.queries.parse import \
Check warning on line 90 in archive_query_log/legacy/model/parse.py Codecov / codecov/patch archive_query_log/legacy/model/parse.py#L90 `Added line #L90 was not covered by tests`
	FragmentSegmentQueryParser
	return FragmentSegmentQueryParser(
	url_pattern=pattern(value["url_pattern"], IGNORECASE),
	),
	)
	elif parser_type == "fragment_segment":
	from archive_query_log.legacy.queries.parse import \
Check warning on line 156 in archive_query_log/legacy/model/parse.py Codecov / codecov/patch archive_query_log/legacy/model/parse.py#L156 `Added line #L156 was not covered by tests`
	FragmentSegmentPageOffsetParser
	return FragmentSegmentPageOffsetParser(
	url_pattern=pattern(value["url_pattern"], IGNORECASE),
	),
	)
	elif parser_type == "chatnoir":
	from archive_query_log.legacy.results.chatnoir import \
Check warning on line 234 in archive_query_log/legacy/model/parse.py Codecov / codecov/patch archive_query_log/legacy/model/parse.py#L234 `Added line #L234 was not covered by tests`
	ChatNoirResultsParser
	return ChatNoirResultsParser(
	url_pattern=pattern(value["url_pattern"], IGNORECASE),
	if output_path.exists() and not self.overwrite:
	return
	output_path.parent.mkdir(parents=True, exist_ok=True)
	archived_urls: Iterable[ArchivedUrl] = ArchivedUrls(input_path)
Check warning on line 183 in archive_query_log/legacy/queries/parse.py Codecov / codecov/patch archive_query_log/legacy/queries/parse.py#L183 `Added line #L183 was not covered by tests`
	if self.verbose:
	# noinspection PyTypeChecker
	archived_urls = tqdm(
	desc="Parse SERP URLs",
	unit="URL",
	)
	archived_serp_urls_nullable = (
Check warning on line 191 in archive_query_log/legacy/queries/parse.py Codecov / codecov/patch archive_query_log/legacy/queries/parse.py#L191 `Added line #L191 was not covered by tests`
	self._parse_single(archived_url, focused)
	for archived_url in archived_urls
	)
	if archived_serp_url is not None
	)
	output_schema = ArchivedQueryUrl.schema()
	with (output_path.open("wb") as file,
Check warning on line 201 in archive_query_log/legacy/queries/parse.py Codecov / codecov/patch archive_query_log/legacy/queries/parse.py#L201 `Added line #L201 was not covered by tests`
	GzipFile(fileobj=file, mode="wb") as gzip_file,
	text_io_wrapper(gzip_file) as text_file):
	for archived_serp_url in archived_serp_urls:
	focused: bool,
	) -> ArchivedQueryUrl \| None:
	query: str \| None = None
	for query_parser in self.query_parsers:
	query = query_parser.parse(archived_url)
Check warning on line 215 in archive_query_log/legacy/queries/parse.py Codecov / codecov/patch archive_query_log/legacy/queries/parse.py#L214-L215 `Added lines #L214 - L215 were not covered by tests`
	if query is not None:
	break

	return None

	page: int \| None = None
	for page_parser in self.page_parsers:
	page = page_parser.parse(archived_url)
Check warning on line 224 in archive_query_log/legacy/queries/parse.py Codecov / codecov/patch archive_query_log/legacy/queries/parse.py#L223-L224 `Added lines #L223 - L224 were not covered by tests`
	if page is not None:
	break

	return None

	offset: int \| None = None
	for offset_parser in self.offset_parsers:
	offset = offset_parser.parse(archived_url)
Check warning on line 233 in archive_query_log/legacy/queries/parse.py Codecov / codecov/patch archive_query_log/legacy/queries/parse.py#L232-L233 `Added lines #L232 - L233 were not covered by tests`
	if offset is not None:
	break

	]

	if cdx_page is not None:
	if domain is None:
	raise RuntimeError(
Check warning on line 284 in archive_query_log/legacy/queries/parse.py Codecov / codecov/patch archive_query_log/legacy/queries/parse.py#L283-L284 `Added lines #L283 - L284 were not covered by tests`
	"Domain must be specified when page is specified.")
	if len(domain_paths) < 1:
	raise RuntimeError(
Check warning on line 287 in archive_query_log/legacy/queries/parse.py Codecov / codecov/patch archive_query_log/legacy/queries/parse.py#L286-L287 `Added lines #L286 - L287 were not covered by tests`
	"There must be exactly one domain path.")
	cdx_page_paths = [domain_paths[0] / f"{cdx_page:010}.jsonl.gz"]
	else:
	domain: str \| None = None,
	cdx_page: int \| None = None,
	):
	pages_list: Sequence[_CdxPage] = self._service_pages(
Check warning on line 320 in archive_query_log/legacy/queries/parse.py Codecov / codecov/patch archive_query_log/legacy/queries/parse.py#L320 `Added line #L320 was not covered by tests`
	data_directory=data_directory,
	focused=focused,
	service=service,
	cdx_page=cdx_page,
	)

	if len(pages_list) == 0:
Check warning on line 328 in archive_query_log/legacy/queries/parse.py Codecov / codecov/patch archive_query_log/legacy/queries/parse.py#L328 `Added line #L328 was not covered by tests`
	return

	pages: Iterable[_CdxPage] = pages_list
	if len(pages_list) > 1:
Check warning on line 332 in archive_query_log/legacy/queries/parse.py Codecov / codecov/patch archive_query_log/legacy/queries/parse.py#L331-L332 `Added lines #L331 - L332 were not covered by tests`
	# noinspection PyTypeChecker
	pages = tqdm(
	pages,
	try:
	service = Service.schema(unknown="exclude").load(service_dict)
	services += [(service.name, service)]
	except ValidationError as e:
	if not ignore_parsing_errors:
Check warning on line 22 in archive_query_log/legacy/services/__init__.py Codecov / codecov/patch archive_query_log/legacy/services/__init__.py#L21-L22 `Added lines #L21 - L22 were not covered by tests`
	raise ValueError(
	f"Could not parse service {service_dict}") from e
	if not ignore_parsing_errors:
	self._check_urls_path()

	def _check_urls_path(self):
	if not self.path.exists() or not self.path.is_file():
	raise ValueError(
Check warning on line 26 in archive_query_log/legacy/urls/iterable.py Codecov / codecov/patch archive_query_log/legacy/urls/iterable.py#L25-L26 `Added lines #L25 - L26 were not covered by tests`
	f"URLs path must be a file: {self.path}"
	)

	def __len__(self) -> int:
	with (self.path.open("rb") as file,
Check warning on line 31 in archive_query_log/legacy/urls/iterable.py Codecov / codecov/patch archive_query_log/legacy/urls/iterable.py#L31 `Added line #L31 was not covered by tests`
	GzipFile(fileobj=file, mode="rb") as gzip_file):
	return count_lines(gzip_file)
Check warning on line 33 in archive_query_log/legacy/urls/iterable.py Codecov / codecov/patch archive_query_log/legacy/urls/iterable.py#L33 `Added line #L33 was not covered by tests`

	def __iter__(self) -> Iterator[ArchivedUrl]:
	schema = ArchivedUrl.schema()
	with (self.path.open("rb") as file,
Check warning on line 37 in archive_query_log/legacy/urls/iterable.py Codecov / codecov/patch archive_query_log/legacy/urls/iterable.py#L36-L37 `Added lines #L36 - L37 were not covered by tests`
	GzipFile(fileobj=file, mode="rb") as gzip_file,
	text_io_wrapper(gzip_file) as text_file):
	for line in text_file:
	url = schema.loads(line)
	if isinstance(url, list):
	raise ValueError(f"Expected one URL per line: {line}")
	yield url
Check warning on line 44 in archive_query_log/legacy/urls/iterable.py Codecov / codecov/patch archive_query_log/legacy/urls/iterable.py#L40-L44 `Added lines #L40 - L44 were not covered by tests`


	def text_io_wrapper(file: IO[bytes] \| IOBase) -> IO[str]:
	return TextIOWrapper(file) # type: ignore
Check warning on line 19 in archive_query_log/legacy/util/text.py Codecov / codecov/patch archive_query_log/legacy/util/text.py#L19 `Added line #L19 was not covered by tests`