From d2940323474679620f7ce11516156cfe5f200093 Mon Sep 17 00:00:00 2001 From: Scott Walsh Date: Sat, 2 Mar 2024 10:35:24 -0400 Subject: [PATCH 1/4] helper utility for saving a Markdown file 'save' can be used to save a Markdown file, with optional frontmatter and additional tags. By default, if set, `FABRIC_FRONTMATTER_TAGS` will be placed into the file as it is written. These tags and front matter are suppressed from STDOUT, which can be piped into other patterns or programs with no ill effects. This strives to be a version of `tee` that is enhanced for personal knowledge systems that use frontmatter. --- helpers/save | 109 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100755 helpers/save diff --git a/helpers/save b/helpers/save new file mode 100755 index 0000000..7b2c443 --- /dev/null +++ b/helpers/save @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +import argparse +import os +import sys +from datetime import datetime + +from dotenv import load_dotenv + +DEFAULT_CONFIG = "~/.config/fabric/.env" +PATH_KEY = "FABRIC_OUTPUT_PATH" +FM_KEY = "FABRIC_FRONTMATTER_TAGS" +DATE_FORMAT = "%Y-%m-%d" +load_dotenv(os.path.expanduser(DEFAULT_CONFIG)) + + +def main(tag, tags, fabric): + out = os.getenv(PATH_KEY) + if out is None: + print(f"'{PATH_KEY}' not set in {DEFAULT_CONFIG} or in your environment.") + sys.exit(1) + + out = os.path.expanduser(out) + + if not os.path.isdir(out): + print(f"'{out}' does not exist. Create it and try again.") + sys.exit(1) + + if not out.endswith("/"): + out += "/" + + if len(sys.argv) < 2: + print(f"'{sys.argv[0]}' takes a single argument to tag your summary") + sys.exit(1) + + yyyymmdd = datetime.now().strftime(DATE_FORMAT) + target = f"{out}{yyyymmdd}-{tag}.md" + + # don't clobber existing files- add an incremented number to the end instead + would_clobber = True + inc = 0 + while would_clobber: + if inc > 0: + target = f"{out}{yyyymmdd}-{tag}-{inc}.md" + if os.path.exists(target): + inc += 1 + else: + would_clobber = False + + # YAML frontmatter stubs for things like Obsidian + # Prevent a NoneType ending up in the tags + frontmatter_tags = "" + if fabric: + frontmatter_tags = os.getenv(FM_KEY) + + with open(target, "w") as fp: + if frontmatter_tags or len(tags) != 0: + fp.write("---\n") + now = datetime.now().strftime(f"{DATE_FORMAT} %H:%M") + fp.write(f"generation_date: {now}\n") + fp.write(f"tags: {frontmatter_tags} {tag} {' '.join(tags)}\n") + fp.write("---\n") + + # function like 'tee' and split the output to a file and STDOUT + for line in sys.stdin: + print(line, end="") + fp.write(line) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description=( + 'save: a "tee-like" utility to pipeline saving of content, ' + "while keeping the output stream intact. Can optionally generate " + '"frontmatter" for PKM utilities like Obsidian via the ' + '"FABRIC_FRONTMATTER" environment variable' + ) + ) + parser.add_argument( + "stub", + nargs="?", + help=( + "stub to describe your content. Use quotes if you have spaces. " + "Resulting format is YYYY-MM-DD-stub.md by default" + ), + ) + parser.add_argument( + "-t,", + "--tag", + required=False, + action="append", + default=[], + help=( + "add an additional frontmatter tag. Use this argument multiple times" + "for multiple tags" + ), + ) + parser.add_argument( + "-n", + "--nofabric", + required=False, + action="store_false", + help="don't use the fabric tags, only use tags from --tag", + ) + args = parser.parse_args() + + if args.stub: + main(args.stub, args.tag, args.nofabric) + else: + parser.print_help() From 65829c5c841546262e063abda8b6522f129f6645 Mon Sep 17 00:00:00 2001 From: Scott Walsh Date: Sun, 3 Mar 2024 17:11:51 -0400 Subject: [PATCH 2/4] Update design pattern and docs --- helpers/README.md | 38 ++++++++++++++++++++++++++++++++++++++ helpers/{save => save.py} | 20 ++++++++++++++++---- pyproject.toml | 1 + 3 files changed, 55 insertions(+), 4 deletions(-) rename helpers/{save => save.py} (89%) diff --git a/helpers/README.md b/helpers/README.md index d34dbe3..336e43f 100644 --- a/helpers/README.md +++ b/helpers/README.md @@ -49,4 +49,42 @@ positional arguments: options: -h, --help show this help message and exit + +## save + +`save` is a "tee-like" utility to pipeline saving of content, while keeping the output stream intact. Can optionally generate "frontmatter" for PKM utilities like Obsidian via the +"FABRIC_FRONTMATTER" environment variable + +### Environment Variables + +If you'd like to default variables, set them in `~/.config/fabric/.env`. `FABRIC_OUTPUT_PATH` needs to be set so `save` where to write. `FABRIC_FRONTMATTER_TAGS` is optional, but useful for tracking how tags have entered your PKM, if that's important to you. + +### usage +```bash +usage: save [-h] [-t, TAG] [-n] [-s] [stub] + +save: a "tee-like" utility to pipeline saving of content, while keeping the output stream intact. Can optionally generate "frontmatter" for PKM utilities like Obsidian via the +"FABRIC_FRONTMATTER" environment variable + +positional arguments: + stub stub to describe your content. Use quotes if you have spaces. Resulting format is YYYY-MM-DD-stub.md by default + +options: + -h, --help show this help message and exit + -t, TAG, --tag TAG add an additional frontmatter tag. Use this argument multiple timesfor multiple tags + -n, --nofabric don't use the fabric tags, only use tags from --tag + -s, --silent don't use STDOUT for output, only save to the file +``` +### example + +```bash +echo test | save --tag extra-tag stub-for-name +test + +$ cat ~/obsidian/Fabric/2024-03-02-stub-for-name.md +--- +generation_date: 2024-03-02 10:43 +tags: fabric-extraction stub-for-name extra-tag +--- +test ``` diff --git a/helpers/save b/helpers/save.py similarity index 89% rename from helpers/save rename to helpers/save.py index 7b2c443..9c67744 100755 --- a/helpers/save +++ b/helpers/save.py @@ -13,7 +13,7 @@ DATE_FORMAT = "%Y-%m-%d" load_dotenv(os.path.expanduser(DEFAULT_CONFIG)) -def main(tag, tags, fabric): +def main(tag, tags, silent, fabric): out = os.getenv(PATH_KEY) if out is None: print(f"'{PATH_KEY}' not set in {DEFAULT_CONFIG} or in your environment.") @@ -62,11 +62,12 @@ def main(tag, tags, fabric): # function like 'tee' and split the output to a file and STDOUT for line in sys.stdin: - print(line, end="") + if not silent: + print(line, end="") fp.write(line) -if __name__ == "__main__": +def cli(): parser = argparse.ArgumentParser( description=( 'save: a "tee-like" utility to pipeline saving of content, ' @@ -101,9 +102,20 @@ if __name__ == "__main__": action="store_false", help="don't use the fabric tags, only use tags from --tag", ) + parser.add_argument( + "-s", + "--silent", + required=False, + action="store_true", + help="don't use STDOUT for output, only save to the file", + ) args = parser.parse_args() if args.stub: - main(args.stub, args.tag, args.nofabric) + main(args.stub, args.tag, args.silent, args.nofabric) else: parser.print_help() + + +if __name__ == "__main__": + cli() diff --git a/pyproject.toml b/pyproject.toml index ae76970..457181e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,3 +62,4 @@ fabric-api = 'installer:run_api_server' fabric-webui = 'installer:run_webui_server' ts = 'helpers.ts:main' yt = 'helpers.yt:main' +save = 'helpers.save:cli' From 6bbb0a5f2f6482c47424f7c094393a7db0b3aaed Mon Sep 17 00:00:00 2001 From: Scott Walsh Date: Sun, 3 Mar 2024 17:14:39 -0400 Subject: [PATCH 3/4] Use exception messages for a better chance at debugging --- helpers/yt.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/helpers/yt.py b/helpers/yt.py index a473d96..a29e612 100644 --- a/helpers/yt.py +++ b/helpers/yt.py @@ -54,7 +54,7 @@ def main_function(url, options): for item in transcript_list]) transcript_text = transcript_text.replace('\n', ' ') except Exception as e: - transcript_text = "Transcript not available." + transcript_text = f"Transcript not available. ({e})" # Output based on options if options.duration: @@ -70,12 +70,12 @@ def main_function(url, options): # Print JSON object print(json.dumps(output)) except HttpError as e: - print("Error: Failed to access YouTube API. Please check your YOUTUBE_API_KEY and ensure it is valid.") + print(f"Error: Failed to access YouTube API. Please check your YOUTUBE_API_KEY and ensure it is valid: {e}") def main(): parser = argparse.ArgumentParser( - description='vm (video meta) extracts metadata about a video, such as the transcript and the video\'s duration. By Daniel Miessler.') + description='yt (video meta) extracts metadata about a video, such as the transcript and the video\'s duration. By Daniel Miessler.') parser.add_argument('url', nargs='?', help='YouTube video URL') parser.add_argument('--duration', action='store_true', help='Output only the duration') From 573723cd9a04c27bc70b69363076fa58091565ed Mon Sep 17 00:00:00 2001 From: Scott Walsh Date: Sun, 3 Mar 2024 17:21:16 -0400 Subject: [PATCH 4/4] move usage block --- helpers/README.md | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/helpers/README.md b/helpers/README.md index 336e43f..f505a20 100644 --- a/helpers/README.md +++ b/helpers/README.md @@ -6,6 +6,20 @@ These are helper tools to work with Fabric. Examples include things like getting `yt` is a command that uses the YouTube API to pull transcripts, get video duration, and other functions. It's primary function is to get a transcript from a video that can then be stitched (piped) into other Fabric Patterns. +```bash +usage: yt [-h] [--duration] [--transcript] [url] + +vm (video meta) extracts metadata about a video, such as the transcript and the video's duration. By Daniel Miessler. + +positional arguments: + url YouTube video URL + +options: + -h, --help show this help message and exit + --duration Output only the duration + --transcript Output only the transcript +``` + ## ts (Audio transcriptions) 'ts' is a command that uses the OpenApi Whisper API to transcribe audio files. Due to the context window, this tool uses pydub to split the files into 10 minute segments. for more information on pydub, please refer https://github.com/jiaaro/pydub @@ -24,19 +38,6 @@ windows: download instructions https://www.ffmpeg.org/download.html ``` -```bash -usage: yt [-h] [--duration] [--transcript] [url] - -vm (video meta) extracts metadata about a video, such as the transcript and the video's duration. By Daniel Miessler. - -positional arguments: - url YouTube video URL - -options: - -h, --help show this help message and exit - --duration Output only the duration - --transcript Output only the transcript -``` ```bash ts -h