Browse Source

Merge remote-tracking branch 'upstream/main' into sanjaypsemwal-feature-branch

pull/18/head
Sanjay Semwal 6 months ago
parent
commit
fba0a462e7
  1. 3
      .gitignore
  2. 15
      README.md
  3. BIN
      business.jpg
  4. 54
      diagnostics.ipynb
  5. 419
      diagnostics.py
  6. 4
      environment.yml
  7. 352
      extras/trading/prototype_trader.ipynb
  8. 725
      extras/trading/trades_claude.py
  9. 534
      extras/trading/trades_gemini.py
  10. 884
      extras/trading/trades_gpt-4o.py
  11. BIN
      important.jpg
  12. 3
      requirements.txt
  13. BIN
      resources.jpg
  14. 10
      week1/Intermediate Python.ipynb
  15. 88
      week1/day1.ipynb
  16. 44
      week1/day2 EXERCISE.ipynb
  17. 62
      week1/day5.ipynb
  18. 419
      week1/diagnostics.py
  19. 81
      week1/troubleshooting.ipynb
  20. 689
      week2/community-contributions/day1-azure-aws-ollama.ipynb
  21. 92
      week2/day1.ipynb
  22. 83
      week2/day2.ipynb
  23. 18
      week2/day3.ipynb
  24. 52
      week4/day3.ipynb
  25. 34
      week4/day4.ipynb
  26. 39
      week8/day1.ipynb
  27. 7
      week8/day5.ipynb
  28. 2
      week8/price_is_right.py

3
.gitignore vendored

@ -175,3 +175,6 @@ products_vectorstore/
# ignore gradio private files # ignore gradio private files
.gradio .gradio
/.gradio /.gradio
# ignore diagnostics reports
**/report.txt

15
README.md

@ -76,3 +76,18 @@ The charges for the exercsies in this course should always be quite low, but if
3. During week 7, look out for my instructions for using the cheaper dataset 3. During week 7, look out for my instructions for using the cheaper dataset
Please do message me or email me at ed@edwarddonner.com if this doesn't work or if I can help with anything. I can't wait to hear how you get on. Please do message me or email me at ed@edwarddonner.com if this doesn't work or if I can help with anything. I can't wait to hear how you get on.
<table style="margin: 0; text-align: left;">
<tr>
<td style="width: 150px; height: 150px; vertical-align: middle;">
<img src="resources.jpg" width="150" height="150" style="display: block;" />
</td>
<td>
<h2 style="color:#f71;">Other resources</h2>
<span style="color:#f71;">I've put together this webpage with useful resources for the course. This includes links to all the slides.<br/>
<a href="https://edwarddonner.com/2024/11/13/llm-engineering-resources/">https://edwarddonner.com/2024/11/13/llm-engineering-resources/</a><br/>
Please keep this bookmarked, and I'll continue to add more useful links there over time.
</span>
</td>
</tr>
</table>

BIN
business.jpg

Binary file not shown.

After

Width:  |  Height:  |  Size: 367 KiB

54
diagnostics.ipynb

@ -0,0 +1,54 @@
{
"cells": [
{
"cell_type": "markdown",
"id": "73287ed4-81e3-496a-9e47-f0e8c3770ce9",
"metadata": {},
"source": [
"# Gathering Essential Diagnostic information\n",
"\n",
"## Please run this next cell to gather some important data\n",
"\n",
"Please run the next cell; it should take a minute or so to run (mostly the network test).\n",
"Rhen email me the output of the last cell to ed@edwarddonner.com. \n",
"Alternatively: this will create a file called report.txt - just attach the file to your email."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "ed8056e8-efa2-4b6f-a4bb-e7ceb733c517",
"metadata": {},
"outputs": [],
"source": [
"# Run my diagnostics report to collect key information for debugging\n",
"# Please email me the results. Either copy & paste the output, or attach the file report.txt\n",
"\n",
"!pip install -q requests speedtest-cli psutil setuptools\n",
"from diagnostics import Diagnostics\n",
"Diagnostics().run()"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.10"
}
},
"nbformat": 4,
"nbformat_minor": 5
}

419
diagnostics.py

@ -0,0 +1,419 @@
import os
import sys
import platform
import subprocess
import shutil
import time
import ssl
import tempfile
from pathlib import Path
from datetime import datetime
class Diagnostics:
FILENAME = 'report.txt'
def __init__(self):
self.errors = []
self.warnings = []
if os.path.exists(self.FILENAME):
os.remove(self.FILENAME)
def log(self, message):
print(message)
with open(self.FILENAME, 'a', encoding='utf-8') as f:
f.write(message + "\n")
def start(self):
now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
self.log(f"Starting diagnostics at {now}\n")
def end(self):
now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
self.log(f"\n\nCompleted diagnostics at {now}\n")
print("\nPlease send these diagnostics to me at ed@edwarddonner.com")
print(f"Either copy & paste the above output into an email, or attach the file {self.FILENAME} that has been created in this directory.")
def _log_error(self, message):
self.log(f"ERROR: {message}")
self.errors.append(message)
def _log_warning(self, message):
self.log(f"WARNING: {message}")
self.warnings.append(message)
def run(self):
self.start()
self._step1_system_info()
self._step2_check_files()
self._step3_git_repo()
self._step4_check_env_file()
self._step5_anaconda_check()
self._step6_virtualenv_check()
self._step7_network_connectivity()
self._step8_environment_variables()
self._step9_additional_diagnostics()
if self.warnings:
self.log("\n===== Warnings Found =====")
self.log("The following warnings were detected. They might not prevent the program from running but could cause unexpected behavior:")
for warning in self.warnings:
self.log(f"- {warning}")
if self.errors:
self.log("\n===== Errors Found =====")
self.log("The following critical issues were detected. Please address them before proceeding:")
for error in self.errors:
self.log(f"- {error}")
if not self.errors and not self.warnings:
self.log("\n✅ All diagnostics passed successfully!")
self.end()
def _step1_system_info(self):
self.log("===== System Information =====")
try:
system = platform.system()
self.log(f"Operating System: {system}")
if system == "Windows":
release, version, csd, ptype = platform.win32_ver()
self.log(f"Windows Release: {release}")
self.log(f"Windows Version: {version}")
elif system == "Darwin":
release, version, machine = platform.mac_ver()
self.log(f"MacOS Version: {release}")
else:
self.log(f"Platform: {platform.platform()}")
self.log(f"Architecture: {platform.architecture()}")
self.log(f"Machine: {platform.machine()}")
self.log(f"Processor: {platform.processor()}")
try:
import psutil
ram = psutil.virtual_memory()
total_ram_gb = ram.total / (1024 ** 3)
available_ram_gb = ram.available / (1024 ** 3)
self.log(f"Total RAM: {total_ram_gb:.2f} GB")
self.log(f"Available RAM: {available_ram_gb:.2f} GB")
if available_ram_gb < 2:
self._log_warning(f"Low available RAM: {available_ram_gb:.2f} GB")
except ImportError:
self._log_warning("psutil module not found. Cannot determine RAM information.")
total, used, free = shutil.disk_usage(os.path.expanduser("~"))
free_gb = free / (1024 ** 3)
self.log(f"Free Disk Space: {free_gb:.2f} GB")
if free_gb < 5:
self._log_warning(f"Low disk space: {free_gb:.2f} GB free")
except Exception as e:
self._log_error(f"System information check failed: {e}")
def _step2_check_files(self):
self.log("\n===== File System Information =====")
try:
current_dir = os.getcwd()
self.log(f"Current Directory: {current_dir}")
# Check write permissions
test_file = Path(current_dir) / ".test_write_permission"
try:
test_file.touch(exist_ok=True)
test_file.unlink()
self.log("Write permission: OK")
except Exception as e:
self._log_error(f"No write permission in current directory: {e}")
self.log("\nFiles in Current Directory:")
try:
for item in sorted(os.listdir(current_dir)):
self.log(f" - {item}")
except Exception as e:
self._log_error(f"Cannot list directory contents: {e}")
except Exception as e:
self._log_error(f"File system check failed: {e}")
def _step3_git_repo(self):
self.log("\n===== Git Repository Information =====")
try:
result = subprocess.run(['git', 'rev-parse', '--show-toplevel'],
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
if result.returncode == 0:
git_root = result.stdout.strip()
self.log(f"Git Repository Root: {git_root}")
result = subprocess.run(['git', 'rev-parse', 'HEAD'],
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
if result.returncode == 0:
self.log(f"Current Commit: {result.stdout.strip()}")
else:
self._log_warning(f"Could not get current commit: {result.stderr.strip()}")
result = subprocess.run(['git', 'remote', 'get-url', 'origin'],
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
if result.returncode == 0:
self.log(f"Remote Origin: {result.stdout.strip()}")
else:
self._log_warning("No remote 'origin' configured")
else:
self._log_warning("Not a git repository")
except FileNotFoundError:
self._log_warning("Git is not installed or not in PATH")
except Exception as e:
self._log_error(f"Git check failed: {e}")
def _step4_check_env_file(self):
self.log("\n===== Environment File Check =====")
try:
result = subprocess.run(['git', 'rev-parse', '--show-toplevel'],
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
if result.returncode == 0:
git_root = result.stdout.strip()
env_path = os.path.join(git_root, '.env')
if os.path.isfile(env_path):
self.log(f".env file exists at: {env_path}")
try:
with open(env_path, 'r') as f:
has_api_key = any(line.strip().startswith('OPENAI_API_KEY=') for line in f)
if has_api_key:
self.log("OPENAI_API_KEY found in .env file")
else:
self._log_warning("OPENAI_API_KEY not found in .env file")
except Exception as e:
self._log_error(f"Cannot read .env file: {e}")
else:
self._log_warning(".env file not found in project root")
# Check for additional .env files
for root, _, files in os.walk(git_root):
if '.env' in files and os.path.join(root, '.env') != env_path:
self._log_warning(f"Additional .env file found at: {os.path.join(root, '.env')}")
else:
self._log_warning("Git root directory not found. Cannot perform .env file check.")
except FileNotFoundError:
self._log_warning("Git is not installed or not in PATH")
except Exception as e:
self._log_error(f"Environment file check failed: {e}")
def _step5_anaconda_check(self):
self.log("\n===== Anaconda Environment Check =====")
try:
conda_prefix = os.environ.get('CONDA_PREFIX')
if conda_prefix:
self.log("Anaconda environment is active:")
self.log(f"Environment Path: {conda_prefix}")
self.log(f"Environment Name: {os.path.basename(conda_prefix)}")
conda_exe = os.environ.get('CONDA_EXE', 'conda')
result = subprocess.run([conda_exe, '--version'],
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
if result.returncode == 0:
self.log(f"Conda Version: {result.stdout.strip()}")
else:
self._log_warning("Could not determine Conda version")
self._check_python_packages()
else:
self.log("No active Anaconda environment detected")
except Exception as e:
self._log_error(f"Anaconda environment check failed: {e}")
def _step6_virtualenv_check(self):
self.log("\n===== Virtualenv Check =====")
try:
virtual_env = os.environ.get('VIRTUAL_ENV')
if virtual_env:
self.log("Virtualenv is active:")
self.log(f"Environment Path: {virtual_env}")
self.log(f"Environment Name: {os.path.basename(virtual_env)}")
self._check_python_packages()
else:
self.log("No active virtualenv detected")
if not virtual_env and not os.environ.get('CONDA_PREFIX'):
self._log_warning("Neither virtualenv nor Anaconda environment is active")
except Exception as e:
self._log_error(f"Virtualenv check failed: {e}")
def _check_python_packages(self):
self.log("\nPython Environment:")
self.log(f"Python Version: {sys.version}")
self.log(f"Python Executable: {sys.executable}")
required_packages = ['openai', 'python-dotenv', 'requests', 'gradio', 'transformers']
try:
import pkg_resources
installed = {pkg.key: pkg.version for pkg in pkg_resources.working_set}
self.log("\nRequired Package Versions:")
for package in required_packages:
if package in installed:
self.log(f"{package}: {installed[package]}")
else:
self._log_error(f"Required package '{package}' is not installed")
# Check for potentially conflicting packages
problem_pairs = [
('openai', 'openai-python'),
('python-dotenv', 'dotenv')
]
for pkg1, pkg2 in problem_pairs:
if pkg1 in installed and pkg2 in installed:
self._log_warning(f"Potentially conflicting packages: {pkg1} and {pkg2}")
except ImportError:
self._log_error("Could not import 'pkg_resources' to check installed packages")
except Exception as e:
self._log_error(f"Package check failed: {e}")
def _step7_network_connectivity(self):
self.log("\n===== Network Connectivity Check =====")
try:
self.log(f"SSL Version: {ssl.OPENSSL_VERSION}")
import requests
import speedtest # Importing the speedtest-cli library
# Basic connectivity check
urls = [
'https://www.google.com',
'https://www.cloudflare.com'
]
connected = False
for url in urls:
try:
start_time = time.time()
response = requests.get(url, timeout=10)
elapsed_time = time.time() - start_time
response.raise_for_status()
self.log(f"✓ Connected to {url}")
self.log(f" Response time: {elapsed_time:.2f}s")
if elapsed_time > 2:
self._log_warning(f"Slow response from {url}: {elapsed_time:.2f}s")
connected = True
break
except requests.exceptions.RequestException as e:
self._log_warning(f"Failed to connect to {url}: {e}")
else:
self.log("Basic connectivity OK")
if not connected:
self._log_error("Failed to connect to any test URLs")
return
# Bandwidth test using speedtest-cli
self.log("\nPerforming bandwidth test using speedtest-cli...")
try:
st = speedtest.Speedtest()
st.get_best_server()
download_speed = st.download() # Bits per second
upload_speed = st.upload() # Bits per second
download_mbps = download_speed / 1e6 # Convert to Mbps
upload_mbps = upload_speed / 1e6
self.log(f"Download speed: {download_mbps:.2f} Mbps")
self.log(f"Upload speed: {upload_mbps:.2f} Mbps")
if download_mbps < 1:
self._log_warning("Download speed is low")
if upload_mbps < 0.5:
self._log_warning("Upload speed is low")
except speedtest.ConfigRetrievalError:
self._log_error("Failed to retrieve speedtest configuration")
except Exception as e:
self._log_warning(f"Bandwidth test failed: {e}")
except ImportError:
self._log_error("Required packages are not installed. Please install them using 'pip install requests speedtest-cli'")
except Exception as e:
self._log_error(f"Network connectivity check failed: {e}")
def _step8_environment_variables(self):
self.log("\n===== Environment Variables Check =====")
try:
# Check Python paths
pythonpath = os.environ.get('PYTHONPATH')
if pythonpath:
self.log("\nPYTHONPATH:")
for path in pythonpath.split(os.pathsep):
self.log(f" - {path}")
else:
self.log("\nPYTHONPATH is not set.")
self.log("\nPython sys.path:")
for path in sys.path:
self.log(f" - {path}")
# Check OPENAI_API_KEY
from dotenv import load_dotenv
load_dotenv()
api_key = os.environ.get('OPENAI_API_KEY')
if api_key:
self.log("OPENAI_API_KEY is set after calling load_dotenv()")
if not api_key.startswith('sk-proj-') or len(api_key)<12:
self._log_warning("OPENAI_API_KEY format looks incorrect after calling load_dotenv()")
else:
self._log_warning("OPENAI_API_KEY environment variable is not set after calling load_dotenv()")
except Exception as e:
self._log_error(f"Environment variables check failed: {e}")
def _step9_additional_diagnostics(self):
self.log("\n===== Additional Diagnostics =====")
try:
# Get the site-packages directory paths
import site
site_packages_paths = site.getsitepackages()
if hasattr(site, 'getusersitepackages'):
site_packages_paths.append(site.getusersitepackages())
# Function to check if a path is within site-packages
def is_in_site_packages(path):
return any(os.path.commonpath([path, sp]) == sp for sp in site_packages_paths)
# Check for potential name conflicts in the current directory and sys.path
conflict_names = ['openai.py', 'dotenv.py']
# Check current directory
current_dir = os.getcwd()
for name in conflict_names:
conflict_path = os.path.join(current_dir, name)
if os.path.isfile(conflict_path):
self._log_warning(f"Found '{name}' in the current directory, which may cause import conflicts: {conflict_path}")
# Check sys.path directories
for path in sys.path:
if not path or is_in_site_packages(path):
continue # Skip site-packages and empty paths
for name in conflict_names:
conflict_file = os.path.join(path, name)
if os.path.isfile(conflict_file):
self._log_warning(f"Potential naming conflict: {conflict_file}")
# Check temp directory
try:
with tempfile.NamedTemporaryFile() as tmp:
self.log(f"Temp directory is writable: {os.path.dirname(tmp.name)}")
except Exception as e:
self._log_error(f"Cannot write to temp directory: {e}")
except Exception as e:
self._log_error(f"Additional diagnostics failed: {e}")
if __name__ == "__main__":
diagnostics = Diagnostics()
diagnostics.run()

4
environment.yml

@ -45,4 +45,6 @@ dependencies:
- gensim - gensim
- modal - modal
- ollama - ollama
- psutil
- setuptools
- speedtest-cli

352
extras/trading/prototype_trader.ipynb

@ -0,0 +1,352 @@
{
"cells": [
{
"cell_type": "markdown",
"id": "46d90d45-2d19-49c7-b853-6809dc417ea7",
"metadata": {},
"source": [
"# Extra Project - Trading Code Generator\n",
"\n",
"This is an example extra project to show fine-tuning in action, and applied to code generation.\n",
"\n",
"## Project Brief\n",
"\n",
"Build a prototype LLM that can generate example code to suggest trading decisions to buy or sell stocks!\n",
"\n",
"I generated test data using frontier models, in the other files in this directory. Use this to train an open source code model.\n",
"\n",
"<table style=\"margin: 0; text-align: left;\">\n",
" <tr>\n",
" <td style=\"width: 150px; height: 150px; vertical-align: middle;\">\n",
" <img src=\"../../resources.jpg\" width=\"150\" height=\"150\" style=\"display: block;\" />\n",
" </td>\n",
" <td>\n",
" <h2 style=\"color:#f71;\">This project is provided as an extra resource</h2>\n",
" <span style=\"color:#f71;\">It will make most sense after completing Week 7, and might trigger some ideas for your own projects.<br/><br/>\n",
" This is provided without a detailed walk-through; the output from the colab has been saved (see last cell) so you can review the results if you have any problems running yourself.\n",
" </span>\n",
" </td>\n",
" </tr>\n",
"</table>\n",
"<table style=\"margin: 0; text-align: left;\">\n",
" <tr>\n",
" <td style=\"width: 150px; height: 150px; vertical-align: middle;\">\n",
" <img src=\"../../important.jpg\" width=\"150\" height=\"150\" style=\"display: block;\" />\n",
" </td>\n",
" <td>\n",
" <h2 style=\"color:#900;\">Do not use for actual trading decisions!!</h2>\n",
" <span style=\"color:#900;\">It hopefully goes without saying: this project will generate toy trading code that is over-simplified and untrusted.<br/><br/>Please do not make actual trading decisions based on this!</span>\n",
" </td>\n",
" </tr>\n",
"</table>"
]
},
{
"cell_type": "markdown",
"id": "7e2c4bbb-5e8b-4d84-9997-ecb2c349cf54",
"metadata": {},
"source": [
"## First step - generate training data from examples\n",
"\n",
"There are 3 sample python files generated (via multiple queries) by GPT-4o, Claude 3 Opus and Gemini 1.5 Pro. \n",
"\n",
"This notebook creates training data from these files, then converts to the HuggingFace format and uploads to the hub.\n",
"\n",
"Afterwards, we will move to Google Colab to fine-tune."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "16cf3aa2-f407-4b95-8b9e-c3c586f67835",
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"import glob\n",
"import matplotlib.pyplot as plt\n",
"import random\n",
"from datasets import Dataset\n",
"from dotenv import load_dotenv\n",
"from huggingface_hub import login\n",
"import transformers\n",
"from transformers import AutoTokenizer"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "375302b6-b6a7-46ea-a74c-c2400dbd8bbe",
"metadata": {},
"outputs": [],
"source": [
"# Load environment variables in a file called .env\n",
"from datasets import load_dataset, Dataset\n",
"load_dotenv()\n",
"hf_token = os.getenv('HF_TOKEN')\n",
"if hf_token and hf_token.startswith(\"hf_\") and len(hf_token)>5:\n",
" print(\"HuggingFace Token looks good so far\")\n",
"else:\n",
" print(\"Potential problem with HuggingFace token - please check your .env file, and see the Troubleshooting notebook for more\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "8a0c9fff-9eff-42fd-971b-403c99d9b726",
"metadata": {},
"outputs": [],
"source": [
"# Constants\n",
"\n",
"DATASET_NAME = \"trade_code_data\"\n",
"BASE_MODEL = \"Qwen/CodeQwen1.5-7B\""
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "586b07ba-5396-4c34-a696-01c8bc3597a0",
"metadata": {},
"outputs": [],
"source": [
"# A utility method to convert the text contents of a file into a list of methods\n",
"\n",
"def extract_method_bodies(text):\n",
" chunks = text.split('def trade')[1:]\n",
" results = []\n",
" for chunk in chunks:\n",
" lines = chunk.split('\\n')[1:]\n",
" body = '\\n'.join(line for line in lines if line!='\\n')\n",
" results.append(body)\n",
" return results "
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "953422d0-2e75-4d01-862e-6383df54d9e5",
"metadata": {},
"outputs": [],
"source": [
"# Read all .py files and convert into training data\n",
"\n",
"bodies = []\n",
"for filename in glob.glob(\"*.py\"):\n",
" with open(filename, 'r', encoding='utf-8') as file:\n",
" content = file.read()\n",
" extracted = extract_method_bodies(content)\n",
" bodies += extracted\n",
"\n",
"print(f\"Extracted {len(bodies)} trade method bodies\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "836480e9-ba23-4aa3-a7e2-2666884e9a06",
"metadata": {},
"outputs": [],
"source": [
"# Let's look at one\n",
"\n",
"print(random.choice(bodies))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "47b10e7e-a562-4968-af3f-254a9b424ac8",
"metadata": {},
"outputs": [],
"source": [
"# To visualize the lines of code in each \n",
"\n",
"%matplotlib inline\n",
"fig, ax = plt.subplots(1, 1)\n",
"lengths = [len(body.split('\\n')) for body in bodies]\n",
"ax.set_xlabel('Lines of code')\n",
"ax.set_ylabel('Count of training samples');\n",
"_ = ax.hist(lengths, rwidth=0.7, color=\"green\", bins=range(0, max(lengths)))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "03b37f62-679e-4c3d-9e5b-5878a82696e6",
"metadata": {},
"outputs": [],
"source": [
"# Add the prompt to the start of every training example\n",
"\n",
"prompt = \"\"\"\n",
"# tickers is a list of stock tickers\n",
"import tickers\n",
"\n",
"# prices is a dict; the key is a ticker and the value is a list of historic prices, today first\n",
"import prices\n",
"\n",
"# Trade represents a decision to buy or sell a quantity of a ticker\n",
"import Trade\n",
"\n",
"import random\n",
"import numpy as np\n",
"\n",
"def trade():\n",
"\"\"\"\n",
"\n",
"data = [prompt + body for body in bodies]\n",
"print(random.choice(data))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "28fdb82f-3864-4023-8263-547d17571a5c",
"metadata": {},
"outputs": [],
"source": [
"# Distribution of tokens in our dataset\n",
"\n",
"tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL, trust_remote_code=True)\n",
"tokenized_data = [tokenizer.encode(each) for each in data]\n",
"token_counts = [len(tokens) for tokens in tokenized_data]\n",
"\n",
"%matplotlib inline\n",
"fig, ax = plt.subplots(1, 1)\n",
"ax.set_xlabel('Number of tokens')\n",
"ax.set_ylabel('Count of training samples');\n",
"_ = ax.hist(token_counts, rwidth=0.7, color=\"purple\", bins=range(0, max(token_counts), 20))"
]
},
{
"cell_type": "markdown",
"id": "b4eb73b0-88ef-4aeb-8e5b-fe7050109ba0",
"metadata": {},
"source": [
"# Enforcing a maximum token length\n",
"\n",
"We need to specify a maximum number of tokens when we fine-tune.\n",
"\n",
"Let's pick a cut-off, and only keep training data points that fit within this number of tokens,"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "ffb0d55c-5602-4518-b811-fa385c0959a7",
"metadata": {},
"outputs": [],
"source": [
"CUTOFF = 320\n",
"truncated = len([tokens for tokens in tokenized_data if len(tokens) > CUTOFF])\n",
"percentage = truncated/len(tokenized_data)*100\n",
"print(f\"With cutoff at {CUTOFF}, we truncate {truncated} datapoints which is {percentage:.1f}% of the dataset\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "7064ef0a-7b07-4f24-a580-cbef2c5e1f2f",
"metadata": {},
"outputs": [],
"source": [
"# Let's only keep datapoints that wouldn't get truncated\n",
"\n",
"filtered_data = [datapoint for datapoint in data if len(tokenizer.encode(datapoint))<=CUTOFF]\n",
"print(f\"After e now have {len(filtered_data)} datapoints\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "fb2bb067-2bd3-498b-9fc8-5e8186afbe27",
"metadata": {},
"outputs": [],
"source": [
"# Mix up the data\n",
"\n",
"random.seed(42)\n",
"random.shuffle(filtered_data)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "26713fb9-765f-4524-b9db-447e97686d1a",
"metadata": {},
"outputs": [],
"source": [
"# I don't make a Training / Test split - if we had more training data, we would!\n",
"\n",
"dataset = Dataset.from_dict({'text':filtered_data})"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "bfabba27-ef47-46a8-a26b-4d650ae3b193",
"metadata": {},
"outputs": [],
"source": [
"login(hf_token)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "55b595cd-2df7-4be4-aec1-0667b17d36f1",
"metadata": {},
"outputs": [],
"source": [
"# Push your dataset to your hub\n",
"# I've also pushed the data to my account and made it public, which you can use from the colab below\n",
"\n",
"dataset.push_to_hub(DATASET_NAME, private=True)"
]
},
{
"cell_type": "markdown",
"id": "4691a025-9800-4e97-a20f-a65f102401f1",
"metadata": {},
"source": [
"## And now to head over to a Google Colab for fine-tuning in the cloud\n",
"\n",
"Follow this link for the Colab:\n",
"\n",
"https://colab.research.google.com/drive/1wry2-4AGw-U7K0LQ_jEgduoTQqVIvo1x?usp=sharing\n",
"\n",
"I've also saved this Colab with output included, so you can see the results without needing to train if you'd prefer.\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "04a6c3e0-a2e6-4115-a01a-45e79dfdb730",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.10"
}
},
"nbformat": 4,
"nbformat_minor": 5
}

725
extras/trading/trades_claude.py

@ -0,0 +1,725 @@
# tickers is a list of stock tickers
import tickers
# prices is a dict; the key is a ticker and the value is a list of historic prices, today first
import prices
# Trade represents a decision to buy or sell a quantity of a ticker
import Trade
import random
import numpy as np
def trade2():
# Buy if the current price is lower than the average of the last 5 days
trades = []
for ticker in tickers:
if prices[ticker][0] < np.mean(prices[ticker][1:6]):
quantity = random.randrange(1, 100)
trades.append(Trade(ticker, quantity))
return trades
def trade3():
# Sell if the current price is higher than the average of the last 10 days
trades = []
for ticker in tickers:
if prices[ticker][0] > np.mean(prices[ticker][1:11]):
quantity = random.randrange(-100, -1)
trades.append(Trade(ticker, quantity))
return trades
def trade4():
# Buy if the current price is the lowest in the last 3 days
trades = []
for ticker in tickers:
if prices[ticker][0] == min(prices[ticker][:3]):
quantity = random.randrange(1, 100)
trades.append(Trade(ticker, quantity))
return trades
def trade5():
# Sell if the current price is the highest in the last 3 days
trades = []
for ticker in tickers:
if prices[ticker][0] == max(prices[ticker][:3]):
quantity = random.randrange(-100, -1)
trades.append(Trade(ticker, quantity))
return trades
def trade6():
# Buy if the current price is higher than the previous day's price
trades = []
for ticker in tickers:
if prices[ticker][0] > prices[ticker][1]:
quantity = random.randrange(1, 100)
trades.append(Trade(ticker, quantity))
return trades
def trade7():
# Sell if the current price is lower than the previous day's price
trades = []
for ticker in tickers:
if prices[ticker][0] < prices[ticker][1]:
quantity = random.randrange(-100, -1)
trades.append(Trade(ticker, quantity))
return trades
def trade8():
# Buy if the current price is higher than the average of the last 20 days
trades = []
for ticker in tickers:
if prices[ticker][0] > np.mean(prices[ticker][1:21]):
quantity = random.randrange(1, 100)
trades.append(Trade(ticker, quantity))
return trades
def trade9():
# Sell if the current price is lower than the average of the last 20 days
trades = []
for ticker in tickers:
if prices[ticker][0] < np.mean(prices[ticker][1:21]):
quantity = random.randrange(-100, -1)
trades.append(Trade(ticker, quantity))
return trades
def trade10():
# Buy if the current price is higher than the highest price in the last 5 days
trades = []
for ticker in tickers:
if prices[ticker][0] > max(prices[ticker][1:6]):
quantity = random.randrange(1, 100)
trades.append(Trade(ticker, quantity))
return trades
def trade11():
# Sell if the current price is lower than the lowest price in the last 5 days
trades = []
for ticker in tickers:
if prices[ticker][0] < min(prices[ticker][1:6]):
quantity = random.randrange(-100, -1)
trades.append(Trade(ticker, quantity))
return trades
def trade12():
# Long/Short: Buy the best-performing stock and sell the worst-performing stock in the last 10 days
best_ticker = max(tickers, key=lambda x: (prices[x][0] - prices[x][9]) / prices[x][9])
worst_ticker = min(tickers, key=lambda x: (prices[x][0] - prices[x][9]) / prices[x][9])
return [Trade(best_ticker, 100), Trade(worst_ticker, -100)]
def trade13():
# Buy if the 5-day moving average crosses above the 20-day moving average
trades = []
for ticker in tickers:
if np.mean(prices[ticker][:5]) > np.mean(prices[ticker][:20]) and np.mean(prices[ticker][1:6]) <= np.mean(prices[ticker][1:21]):
quantity = random.randrange(1, 100)
trades.append(Trade(ticker, quantity))
return trades
def trade14():
# Sell if the 5-day moving average crosses below the 20-day moving average
trades = []
for ticker in tickers:
if np.mean(prices[ticker][:5]) < np.mean(prices[ticker][:20]) and np.mean(prices[ticker][1:6]) >= np.mean(prices[ticker][1:21]):
quantity = random.randrange(-100, -1)
trades.append(Trade(ticker, quantity))
return trades
def trade15():
# Buy if the current volume is higher than the average volume of the last 10 days
trades = []
for ticker in tickers:
if volumes[ticker][0] > np.mean(volumes[ticker][1:11]):
quantity = random.randrange(1, 100)
trades.append(Trade(ticker, quantity))
return trades
def trade16():
# Sell if the current volume is lower than the average volume of the last 10 days
trades = []
for ticker in tickers:
if volumes[ticker][0] < np.mean(volumes[ticker][1:11]):
quantity = random.randrange(-100, -1)
trades.append(Trade(ticker, quantity))
return trades
def trade17():
# Long/Short: Buy the stock with the highest relative strength index (RSI) and sell the stock with the lowest RSI
rsi = {}
for ticker in tickers:
gains = [max(prices[ticker][i] - prices[ticker][i+1], 0) for i in range(13)]
losses = [max(prices[ticker][i+1] - prices[ticker][i], 0) for i in range(13)]
avg_gain = sum(gains) / 14
avg_loss = sum(losses) / 14
rs = avg_gain / avg_loss if avg_loss > 0 else 100
rsi[ticker] = 100 - (100 / (1 + rs))
best_ticker = max(tickers, key=lambda x: rsi[x])
worst_ticker = min(tickers, key=lambda x: rsi[x])
return [Trade(best_ticker, 100), Trade(worst_ticker, -100)]
def trade18():
# Buy if the current price is higher than the 50-day moving average and the 50-day moving average is higher than the 200-day moving average
trades = []
for ticker in tickers:
if prices[ticker][0] > np.mean(prices[ticker][:50]) > np.mean(prices[ticker][:200]):
quantity = random.randrange(1, 100)
trades.append(Trade(ticker, quantity))
return trades
def trade19():
# Sell if the current price is lower than the 50-day moving average and the 50-day moving average is lower than the 200-day moving average
trades = []
for ticker in tickers:
if prices[ticker][0] < np.mean(prices[ticker][:50]) < np.mean(prices[ticker][:200]):
quantity = random.randrange(-100, -1)
trades.append(Trade(ticker, quantity))
return trades
def trade20():
# Long/Short: Buy the stock with the highest momentum and sell the stock with the lowest momentum
momentums = {}
for ticker in tickers:
momentums[ticker] = prices[ticker][0] - prices[ticker][19]
best_ticker = max(tickers, key=lambda x: momentums[x])
worst_ticker = min(tickers, key=lambda x: momentums[x])
return [Trade(best_ticker, 100), Trade(worst_ticker, -100)]
def trade21():
# Buy if the current price is higher than the upper Bollinger Band
trades = []
for ticker in tickers:
sma = np.mean(prices[ticker][:20])
std = np.std(prices[ticker][:20])
upper_band = sma + 2 * std
if prices[ticker][0] > upper_band:
quantity = random.randrange(1, 100)
trades.append(Trade(ticker, quantity))
return trades
def trade22():
# Sell if the current price is lower than the lower Bollinger Band
trades = []
for ticker in tickers:
sma = np.mean(prices[ticker][:20])
std = np.std(prices[ticker][:20])
lower_band = sma - 2 * std
if prices[ticker][0] < lower_band:
quantity = random.randrange(-100, -1)
trades.append(Trade(ticker, quantity))
return trades
def trade23():
# Buy if the current volatility is higher than the average volatility of the last 10 days
trades = []
for ticker in tickers:
volatility = np.std(prices[ticker][:10])
avg_volatility = np.mean([np.std(prices[ticker][i:i+10]) for i in range(10)])
if volatility > avg_volatility:
quantity = random.randrange(1, 100)
trades.append(Trade(ticker, quantity))
return trades
def trade24():
# Sell if the current volatility is lower than the average volatility of the last 10 days
trades = []
for ticker in tickers:
volatility = np.std(prices[ticker][:10])
avg_volatility = np.mean([np.std(prices[ticker][i:i+10]) for i in range(10)])
if volatility < avg_volatility:
quantity = random.randrange(-100, -1)
trades.append(Trade(ticker, quantity))
return trades
def trade25():
# Long/Short: Buy the stock with the lowest volatility and sell the stock with the highest volatility
volatilities = {}
for ticker in tickers:
volatilities[ticker] = np.std(prices[ticker][:10])
best_ticker = min(tickers, key=lambda x: volatilities[x])
worst_ticker = max(tickers, key=lambda x: volatilities[x])
return [Trade(best_ticker, 100), Trade(worst_ticker, -100)]
def trade26():
# Buy if the current price is higher than the 20-day exponential moving average (EMA)
trades = []
for ticker in tickers:
ema = prices[ticker][0]
multiplier = 2 / (20 + 1)
for i in range(1, 20):
ema = (prices[ticker][i] - ema) * multiplier + ema
if prices[ticker][0] > ema:
quantity = random.randrange(1, 100)
trades.append(Trade(ticker, quantity))
return trades
def trade27():
# Sell if the current price is lower than the 20-day exponential moving average (EMA)
trades = []
for ticker in tickers:
ema = prices[ticker][0]
multiplier = 2 / (20 + 1)
for i in range(1, 20):
ema = (prices[ticker][i] - ema) * multiplier + ema
if prices[ticker][0] < ema:
quantity = random.randrange(-100, -1)
trades.append(Trade(ticker, quantity))
return trades
def trade28():
# Buy if the current price is higher than the upper Keltner Channel
trades = []
for ticker in tickers:
ema = prices[ticker][0]
multiplier = 2 / (20 + 1)
for i in range(1, 20):
ema = (prices[ticker][i] - ema) * multiplier + ema
atr = np.mean([np.max(prices[ticker][i:i+10]) - np.min(prices[ticker][i:i+10]) for i in range(10)])
upper_channel = ema + 2 * atr
if prices[ticker][0] > upper_channel:
quantity = random.randrange(1, 100)
trades.append(Trade(ticker, quantity))
return trades
def trade29():
# Sell if the current price is lower than the lower Keltner Channel
trades = []
for ticker in tickers:
ema = prices[ticker][0]
multiplier = 2 / (20 + 1)
for i in range(1, 20):
ema = (prices[ticker][i] - ema) * multiplier + ema
atr = np.mean([np.max(prices[ticker][i:i+10]) - np.min(prices[ticker][i:i+10]) for i in range(10)])
lower_channel = ema - 2 * atr
if prices[ticker][0] < lower_channel:
quantity = random.randrange(-100, -1)
trades.append(Trade(ticker, quantity))
return trades
def trade30():
# Long/Short: Buy the stock with the highest Sharpe ratio and sell the stock with the lowest Sharpe ratio
sharpe_ratios = {}
for ticker in tickers:
returns = [prices[ticker][i] / prices[ticker][i+1] - 1 for i in range(19)]
sharpe_ratios[ticker] = np.mean(returns) / np.std(returns)
best_ticker = max(tickers, key=lambda x: sharpe_ratios[x])
worst_ticker = min(tickers, key=lambda x: sharpe_ratios[x])
return [Trade(best_ticker, 100), Trade(worst_ticker, -100)]
def trade31():
# Buy if the current price is higher than the Ichimoku Cloud conversion line
trades = []
for ticker in tickers:
conversion_line = (np.max(prices[ticker][:9]) + np.min(prices[ticker][:9])) / 2
if prices[ticker][0] > conversion_line:
quantity = random.randrange(1, 100)
trades.append(Trade(ticker, quantity))
return trades
def trade32():
# Buy if the current price is higher than the price 5 days ago
trades = []
for ticker in tickers:
if prices[ticker][0] > prices[ticker][4]:
quantity = random.randrange(1, 100)
trades.append(Trade(ticker, quantity))
return trades
def trade33():
# Sell if the current price is lower than the price 5 days ago
trades = []
for ticker in tickers:
if prices[ticker][0] < prices[ticker][4]:
quantity = random.randrange(-100, -1)
trades.append(Trade(ticker, quantity))
return trades
def trade34():
# Buy if the current price is the highest in the last 15 days
trades = []
for ticker in tickers:
if prices[ticker][0] == max(prices[ticker][:15]):
quantity = random.randrange(1, 100)
trades.append(Trade(ticker, quantity))
return trades
def trade35():
# Sell if the current price is the lowest in the last 15 days
trades = []
for ticker in tickers:
if prices[ticker][0] == min(prices[ticker][:15]):
quantity = random.randrange(-100, -1)
trades.append(Trade(ticker, quantity))
return trades
def trade36():
# Buy if the current price is higher than the 10-day simple moving average (SMA)
trades = []
for ticker in tickers:
sma = np.mean(prices[ticker][:10])
if prices[ticker][0] > sma:
quantity = random.randrange(1, 100)
trades.append(Trade(ticker, quantity))
return trades
def trade37():
# Sell if the current price is lower than the 10-day simple moving average (SMA)
trades = []
for ticker in tickers:
sma = np.mean(prices[ticker][:10])
if prices[ticker][0] < sma:
quantity = random.randrange(-100, -1)
trades.append(Trade(ticker, quantity))
return trades
def trade38():
# Buy if the current price is higher than the highest price in the last 20 days
trades = []
for ticker in tickers:
if prices[ticker][0] > max(prices[ticker][:20]):
quantity = random.randrange(1, 100)
trades.append(Trade(ticker, quantity))
return trades
def trade39():
# Sell if the current price is lower than the lowest price in the last 20 days
trades = []
for ticker in tickers:
if prices[ticker][0] < min(prices[ticker][:20]):
quantity = random.randrange(-100, -1)
trades.append(Trade(ticker, quantity))
return trades
def trade40():
# Buy if the current price is higher than the 50-day SMA
trades = []
for ticker in tickers:
sma = np.mean(prices[ticker][:50])
if prices[ticker][0] > sma:
quantity = random.randrange(1, 100)
trades.append(Trade(ticker, quantity))
return trades
def trade41():
# Sell if the current price is lower than the 50-day SMA
trades = []
for ticker in tickers:
sma = np.mean(prices[ticker][:50])
if prices[ticker][0] < sma:
quantity = random.randrange(-100, -1)
trades.append(Trade(ticker, quantity))
return trades
def trade42():
# Buy if the current price is higher than the previous 2 days (a simple uptrend)
trades = []
for ticker in tickers:
if prices[ticker][0] > prices[ticker][1] > prices[ticker][2]:
quantity = random.randrange(1, 100)
trades.append(Trade(ticker, quantity))
return trades
def trade43():
# Sell if the current price is lower than the previous 2 days (a simple downtrend)
trades = []
for ticker in tickers:
if prices[ticker][0] < prices[ticker][1] < prices[ticker][2]:
quantity = random.randrange(-100, -1)
trades.append(Trade(ticker, quantity))
return trades
def trade44():
# Buy if the current price is higher than the previous day's high (a breakout)
trades = []
for ticker in tickers:
if prices[ticker][0] > max(prices[ticker][1:2]):
quantity = random.randrange(1, 100)
trades.append(Trade(ticker, quantity))
return trades
def trade45():
# Sell if the current price is lower than the previous day's low (a breakdown)
trades = []
for ticker in tickers:
if prices[ticker][0] < min(prices[ticker][1:2]):
quantity = random.randrange(-100, -1)
trades.append(Trade(ticker, quantity))
return trades
def trade46():
# Buy if the current price is above the previous day's high and the previous day was a down day (a potential reversal)
trades = []
for ticker in tickers:
if prices[ticker][0] > max(prices[ticker][1:2]) and prices[ticker][1] < prices[ticker][2]:
quantity = random.randrange(1, 100)
trades.append(Trade(ticker, quantity))
return trades
def trade47():
# Sell if the current price is below the previous day's low and the previous day was an up day (a potential reversal)
trades = []
for ticker in tickers:
if prices[ticker][0] < min(prices[ticker][1:2]) and prices[ticker][1] > prices[ticker][2]:
quantity = random.randrange(-100, -1)
trades.append(Trade(ticker, quantity))
return trades
def trade48():
# Buy if the current price is above the 5-day SMA and the 5-day SMA is above the 10-day SMA (a bullish crossover)
trades = []
for ticker in tickers:
sma5 = np.mean(prices[ticker][:5])
sma10 = np.mean(prices[ticker][:10])
if prices[ticker][0] > sma5 > sma10:
quantity = random.randrange(1, 100)
trades.append(Trade(ticker, quantity))
return trades
def trade49():
# Sell if the current price is below the 5-day SMA and the 5-day SMA is below the 10-day SMA (a bearish crossover)
trades = []
for ticker in tickers:
sma5 = np.mean(prices[ticker][:5])
sma10 = np.mean(prices[ticker][:10])
if prices[ticker][0] < sma5 < sma10:
quantity = random.randrange(-100, -1)
trades.append(Trade(ticker, quantity))
return trades
def trade50():
# Buy if the current price is above the 50-day SMA and the previous price was below the 50-day SMA (a bullish breakthrough)
trades = []
for ticker in tickers:
sma50 = np.mean(prices[ticker][:50])
if prices[ticker][0] > sma50 and prices[ticker][1] < sma50:
quantity = random.randrange(1, 100)
trades.append(Trade(ticker, quantity))
return trades
def trade51():
# Sell if the current price is below the 50-day SMA and the previous price was above the 50-day SMA (a bearish breakthrough)
trades = []
for ticker in tickers:
sma50 = np.mean(prices[ticker][:50])
if prices[ticker][0] < sma50 and prices[ticker][1] > sma50:
quantity = random.randrange(-100, -1)
trades.append(Trade(ticker, quantity))
return trades
def trade52():
# Buy if the current price is more than 2 standard deviations below the 20-day mean (a potential oversold condition)
trades = []
for ticker in tickers:
mean20 = np.mean(prices[ticker][:20])
std20 = np.std(prices[ticker][:20])
if prices[ticker][0] < mean20 - 2 * std20:
quantity = random.randrange(1, 100)
trades.append(Trade(ticker, quantity))
return trades
def trade53():
# Sell if the current price is more than 2 standard deviations above the 20-day mean (a potential overbought condition)
trades = []
for ticker in tickers:
mean20 = np.mean(prices[ticker][:20])
std20 = np.std(prices[ticker][:20])
if prices[ticker][0] > mean20 + 2 * std20:
quantity = random.randrange(-100, -1)
trades.append(Trade(ticker, quantity))
return trades
def trade54():
# Buy if the current price is below the 50-day mean and the 50-day mean is increasing (a potential uptrend)
trades = []
for ticker in tickers:
mean50 = np.mean(prices[ticker][:50])
prev_mean50 = np.mean(prices[ticker][1:51])
if prices[ticker][0] < mean50 and mean50 > prev_mean50:
quantity = random.randrange(1, 100)
trades.append(Trade(ticker, quantity))
return trades
def trade55():
# Sell if the current price is above the 50-day mean and the 50-day mean is decreasing (a potential downtrend)
trades = []
for ticker in tickers:
mean50 = np.mean(prices[ticker][:50])
prev_mean50 = np.mean(prices[ticker][1:51])
if prices[ticker][0] > mean50 and mean50 < prev_mean50:
quantity = random.randrange(-100, -1)
trades.append(Trade(ticker, quantity))
return trades
def trade56():
# Buy if the 5-day mean is above the 50-day mean and the 5-day mean was previously below the 50-day mean (a potential trend change)
trades = []
for ticker in tickers:
mean5 = np.mean(prices[ticker][:5])
mean50 = np.mean(prices[ticker][:50])
prev_mean5 = np.mean(prices[ticker][1:6])
if mean5 > mean50 and prev_mean5 < mean50:
quantity = random.randrange(1, 100)
trades.append(Trade(ticker, quantity))
return trades
def trade57():
# Sell if the 5-day mean is below the 50-day mean and the 5-day mean was previously above the 50-day mean (a potential trend change)
trades = []
for ticker in tickers:
mean5 = np.mean(prices[ticker][:5])
mean50 = np.mean(prices[ticker][:50])
prev_mean5 = np.mean(prices[ticker][1:6])
if mean5 < mean50 and prev_mean5 > mean50:
quantity = random.randrange(-100, -1)
trades.append(Trade(ticker, quantity))
return trades
def trade58():
# Buy the ticker that has had the largest percent decrease over the last 10 days (a potential mean reversion play)
percent_changes = {}
for ticker in tickers:
percent_changes[ticker] = (prices[ticker][0] - prices[ticker][9]) / prices[ticker][9] * 100
worst_ticker = min(tickers, key=lambda x: percent_changes[x])
return [Trade(worst_ticker, 100)]
def trade59():
# Sell the ticker that has had the largest percent increase over the last 10 days (a potential mean reversion play)
percent_changes = {}
for ticker in tickers:
percent_changes[ticker] = (prices[ticker][0] - prices[ticker][9]) / prices[ticker][9] * 100
best_ticker = max(tickers, key=lambda x: percent_changes[x])
return [Trade(best_ticker, -100)]
def trade60():
# Buy if the current price is above the 200-day mean and the 200-day mean is increasing (a potential long-term uptrend)
trades = []
for ticker in tickers:
mean200 = np.mean(prices[ticker][:200])
prev_mean200 = np.mean(prices[ticker][1:201])
if prices[ticker][0] > mean200 and mean200 > prev_mean200:
quantity = random.randrange(1, 100)
trades.append(Trade(ticker, quantity))
return trades
def trade61():
# Sell if the current price is below the 200-day mean and the 200-day mean is decreasing (a potential long-term downtrend)
trades = []
for ticker in tickers:
mean200 = np.mean(prices[ticker][:200])
prev_mean200 = np.mean(prices[ticker][1:201])
if prices[ticker][0] < mean200 and mean200 < prev_mean200:
quantity = random.randrange(-100, -1)
trades.append(Trade(ticker, quantity))
return trades
def trade62():
# Buy if the stock's return is greater than the market's return over the last 5 days
trades = []
for ticker in tickers:
stock_return = (prices[ticker][0] - prices[ticker][4]) / prices[ticker][4]
market_return = (sum(prices[t][0] for t in tickers) - sum(prices[t][4] for t in tickers)) / sum(prices[t][4] for t in tickers)
if stock_return > market_return:
quantity = random.randrange(1, 100)
trades.append(Trade(ticker, quantity))
return trades
def trade63():
# Sell if the stock's return is less than the market's return over the last 5 days
trades = []
for ticker in tickers:
stock_return = (prices[ticker][0] - prices[ticker][4]) / prices[ticker][4]
market_return = (sum(prices[t][0] for t in tickers) - sum(prices[t][4] for t in tickers)) / sum(prices[t][4] for t in tickers)
if stock_return < market_return:
quantity = random.randrange(-100, -1)
trades.append(Trade(ticker, quantity))
return trades
def trade64():
# Buy the stock with the highest relative strength compared to the market over the last 10 days
relative_strengths = {}
for ticker in tickers:
stock_return = prices[ticker][0] / prices[ticker][9]
market_return = sum(prices[t][0] for t in tickers) / sum(prices[t][9] for t in tickers)
relative_strengths[ticker] = stock_return / market_return
best_ticker = max(tickers, key=lambda x: relative_strengths[x])
return [Trade(best_ticker, 100)]
def trade65():
# Sell the stock with the lowest relative strength compared to the market over the last 10 days
relative_strengths = {}
for ticker in tickers:
stock_return = prices[ticker][0] / prices[ticker][9]
market_return = sum(prices[t][0] for t in tickers) / sum(prices[t][9] for t in tickers)
relative_strengths[ticker] = stock_return / market_return
worst_ticker = min(tickers, key=lambda x: relative_strengths[x])
return [Trade(worst_ticker, -100)]
def trade66():
# Buy stocks that have a higher Sharpe ratio than the market over the last 20 days
trades = []
market_returns = [(sum(prices[t][i] for t in tickers) / sum(prices[t][i+1] for t in tickers)) - 1 for i in range(19)]
market_sharpe = np.mean(market_returns) / np.std(market_returns)
for ticker in tickers:
stock_returns = [(prices[ticker][i] / prices[ticker][i+1]) - 1 for i in range(19)]
stock_sharpe = np.mean(stock_returns) / np.std(stock_returns)
if stock_sharpe > market_sharpe:
quantity = random.randrange(1, 100)
trades.append(Trade(ticker, quantity))
return trades
def trade67():
# Sell stocks that have a lower Sharpe ratio than the market over the last 20 days
trades = []
market_returns = [(sum(prices[t][i] for t in tickers) / sum(prices[t][i+1] for t in tickers)) - 1 for i in range(19)]
market_sharpe = np.mean(market_returns) / np.std(market_returns)
for ticker in tickers:
stock_returns = [(prices[ticker][i] / prices[ticker][i+1]) - 1 for i in range(19)]
stock_sharpe = np.mean(stock_returns) / np.std(stock_returns)
if stock_sharpe < market_sharpe:
quantity = random.randrange(-100, -1)
trades.append(Trade(ticker, quantity))
return trades
def trade68():
# Buy stocks that have a higher beta than 1 (they move more than the market)
trades = []
market_returns = [(sum(prices[t][i] for t in tickers) / sum(prices[t][i+1] for t in tickers)) - 1 for i in range(49)]
for ticker in tickers:
stock_returns = [(prices[ticker][i] / prices[ticker][i+1]) - 1 for i in range(49)]
beta = np.cov(stock_returns, market_returns)[0, 1] / np.var(market_returns)
if beta > 1:
quantity = random.randrange(1, 100)
trades.append(Trade(ticker, quantity))
return trades
def trade69():
# Sell stocks that have a lower beta than 1 (they move less than the market)
trades = []
market_returns = [(sum(prices[t][i] for t in tickers) / sum(prices[t][i+1] for t in tickers)) - 1 for i in range(49)]
for ticker in tickers:
stock_returns = [(prices[ticker][i] / prices[ticker][i+1]) - 1 for i in range(49)]
beta = np.cov(stock_returns, market_returns)[0, 1] / np.var(market_returns)
if beta < 1:
quantity = random.randrange(-100, -1)
trades.append(Trade(ticker, quantity))
return trades
def trade70():
# Buy stocks that have a higher percentage of up days than the market over the last 50 days
trades = []
market_up_days = sum(sum(prices[t][i] for t in tickers) > sum(prices[t][i+1] for t in tickers) for i in range(49))
for ticker in tickers:
stock_up_days = sum(prices[ticker][i] > prices[ticker][i+1] for i in range(49))
if stock_up_days > market_up_days:
quantity = random.randrange(1, 100)
trades.append(Trade(ticker, quantity))
return trades
def trade71():
# Sell stocks that have a lower percentage of up days than the market over the last 50 days
trades = []
market_up_days = sum(sum(prices[t][i] for t in tickers) > sum(prices[t][i+1] for t in tickers) for i in range(49))
for ticker in tickers:
stock_up_days = sum(prices[ticker][i] > prices[ticker][i+1] for i in range(49))
if stock_up_days < market_up_days:
quantity = random.randrange(-100, -1)
trades.append(Trade(ticker, quantity))
return trades

534
extras/trading/trades_gemini.py

@ -0,0 +1,534 @@
# tickers is a list of stock tickers
import tickers
# prices is a dict; the key is a ticker and the value is a list of historic prices, today first
import prices
# Trade represents a decision to buy or sell a quantity of a ticker
import Trade
import random
import numpy as np
def trade2():
# Buy the stock with the highest price today
ticker = max(prices, key=lambda t: prices[t][0]) # Find ticker with highest price
return [Trade(ticker, random.randrange(1, 10))] # Buy a small quantity
def trade3():
# Sell the stock with the lowest price today
ticker = min(prices, key=lambda t: prices[t][0])
return [Trade(ticker, random.randrange(-10, -1))]
def trade4():
# Buy the stock with the largest percent increase today
changes = {t: (prices[t][0] - prices[t][1]) / prices[t][1] for t in prices}
ticker = max(changes, key=changes.get)
return [Trade(ticker, random.randrange(1, 10))]
def trade5():
# Sell the stock with the largest percent decrease today
changes = {t: (prices[t][0] - prices[t][1]) / prices[t][1] for t in prices}
ticker = min(changes, key=changes.get)
return [Trade(ticker, random.randrange(-10, -1))]
def trade6():
# Buy the 3 stocks with the highest moving average over the last 5 days
mvgs = {t: np.mean(prices[t][:5]) for t in prices}
top_tickers = sorted(mvgs, key=mvgs.get, reverse=True)[:3]
return [Trade(t, random.randrange(1, 5)) for t in top_tickers]
def trade7():
# Sell the 3 stocks with the lowest moving average over the last 5 days
mvgs = {t: np.mean(prices[t][:5]) for t in prices}
bottom_tickers = sorted(mvgs, key=mvgs.get)[:3]
return [Trade(t, random.randrange(-5, -1)) for t in bottom_tickers]
def trade8():
# Randomly buy or sell a single stock based on a coin flip
ticker = random.choice(tickers)
action = random.choice([-1, 1]) # -1 for sell, 1 for buy
return [Trade(ticker, action * random.randrange(1, 10))]
def trade9():
# Diversify: Buy a small amount of 5 random stocks
chosen_tickers = random.sample(tickers, 5)
return [Trade(t, random.randrange(1, 3)) for t in chosen_tickers]
def trade10():
# Follow the trend: If the market is up today, buy, else sell
market_change = (prices[tickers[0]][0] - prices[tickers[0]][1]) / prices[tickers[0]][1]
action = 1 if market_change > 0 else -1
ticker = random.choice(tickers)
return [Trade(ticker, action * random.randrange(1, 10))]
def trade11():
# Mean Reversion: Buy the 2 stocks that fell the most yesterday, hoping they rebound
yesterday_changes = {t: (prices[t][1] - prices[t][2]) / prices[t][2] for t in prices}
bottom_tickers = sorted(yesterday_changes, key=yesterday_changes.get)[:2]
return [Trade(t, random.randrange(1, 5)) for t in bottom_tickers]
def trade12():
# Momentum: Short the 2 stocks that rose the most yesterday, expecting a pullback
yesterday_changes = {t: (prices[t][1] - prices[t][2]) / prices[t][2] for t in prices}
top_tickers = sorted(yesterday_changes, key=yesterday_changes.get, reverse=True)[:2]
return [Trade(t, random.randrange(-5, -1)) for t in top_tickers]
def trade13():
# Pairs Trading: Long one stock, short another with a similar price history
correlations = np.corrcoef([prices[t] for t in tickers])
i, j = np.unravel_index(np.argmax(correlations), correlations.shape)
return [Trade(tickers[i], 1), Trade(tickers[j], -1)]
def trade14():
# Relative Strength: Go long on the strongest stock, short the weakest
performances = {t: (prices[t][0] - prices[t][-1]) / prices[t][-1] for t in prices}
strongest = max(performances, key=performances.get)
weakest = min(performances, key=performances.get)
return [Trade(strongest, 1), Trade(weakest, -1)]
def trade15():
# Calendar Spread: Buy this month's option, sell next month's (same strike
# This is a simplified representation, as actual option trading is more complex
ticker = random.choice(tickers)
return [Trade(f"{ticker}_OPT_THIS_MONTH", 1), Trade(f"{ticker}_OPT_NEXT_MONTH", -1)]
def trade16():
# Straddle: Buy both a call and put option on the same stock (same strike
ticker = random.choice(tickers)
strike = prices[ticker][0] # Use the current price as a simple strike price
return [Trade(f"{ticker}_CALL_{strike}", 1), Trade(f"{ticker}_PUT_{strike}", 1)]
def trade17():
# Breakout: Buy if a stock breaks above its 52-week high
ticker = random.choice(tickers)
if prices[ticker][0] > max(prices[ticker]):
return [Trade(ticker, random.randrange(1, 10))]
else:
return []
def trade18():
# Volatility: If market volatility is high, sell (expecting it to decrease
market_volatility = np.std([prices[t][0] / prices[t][1] for t in tickers])
if market_volatility > 0.05: # You'd adjust this threshold based on your risk tolerance
ticker = random.choice(tickers)
return [Trade(ticker, random.randrange(-10, -1))]
else:
return []
def trade19():
# Golden Cross: Buy if the short-term moving average crosses above the long-term
ticker = random.choice(tickers)
short_ma = np.mean(prices[ticker][:5])
long_ma = np.mean(prices[ticker][:20])
if short_ma > long_ma and short_ma - long_ma < 0.01: # Small margin to avoid false signals
return [Trade(ticker, random.randrange(1, 10))]
else:
return []
def trade20():
# Death Cross: Sell if the short-term moving average crosses below the long-term
ticker = random.choice(tickers)
short_ma = np.mean(prices[ticker][:5])
long_ma = np.mean(prices[ticker][:20])
if short_ma < long_ma and long_ma - short_ma < 0.01:
return [Trade(ticker, random.randrange(-10, -1))]
else:
return []
def trade21():
# Correlated Pairs Buy: Buy a pair of stocks that have historically moved together
correlations = np.corrcoef([prices[t] for t in tickers])
i, j = np.unravel_index(np.argmax(correlations), correlations.shape)
return [Trade(tickers[i], 1), Trade(tickers[j], 1)]
def trade22():
# Correlated Pairs Sell: Sell a pair of stocks that have historically moved together
correlations = np.corrcoef([prices[t] for t in tickers])
i, j = np.unravel_index(np.argmax(correlations), correlations.shape)
return [Trade(tickers[i], -1), Trade(tickers[j], -1)]
def trade23():
# Contrarian Pairs Buy: Buy a stock that's down while its correlated pair is up
correlations = np.corrcoef([prices[t] for t in tickers])
i, j = np.unravel_index(np.argmax(correlations), correlations.shape)
if prices[tickers[i]][0] < prices[tickers[i]][1] and prices[tickers[j]][0] > prices[tickers[j]][1]:
return [Trade(tickers[i], 1)]
else:
return []
def trade24():
# Contrarian Pairs Sell: Sell a stock that's up while its correlated pair is down
correlations = np.corrcoef([prices[t] for t in tickers])
i, j = np.unravel_index(np.argmax(correlations), correlations.shape)
if prices[tickers[i]][0] > prices[tickers[i]][1] and prices[tickers[j]][0] < prices[tickers[j]][1]:
return [Trade(tickers[i], -1)]
else:
return []
def trade25():
# Correlation Reversal: Buy a stock that's recently become less correlated with the market
# This is a simplified version, you'd likely use a rolling correlation window
market_prices = [prices[t] for t in tickers]
correlations_today = np.corrcoef(market_prices)
correlations_yesterday = np.corrcoef([p[1:] for p in market_prices])
diffs = correlations_today - correlations_yesterday
i, j = np.unravel_index(np.argmin(diffs), diffs.shape)
if i != j: # Ensure we're not comparing a stock to itself
return [Trade(tickers[i], 1)]
else:
return []
def trade26():
# Sector Rotation: Buy the top 2 stocks from the sector that's most correlated with the market
# Assuming you have sector data (e.g., 'sector_map' dict: ticker -> sector)
sector_returns = {s: np.mean([(prices[t][0] - prices[t][1]) / prices[t][1] for t in tickers if sector_map[t] == s]) for s in set(sector_map.values())}
top_sector = max(sector_returns, key=sector_returns.get)
top_tickers_in_sector = sorted([(t, prices[t][0]) for t in tickers if sector_map[t] == top_sector], key=lambda x: x[1], reverse=True)[:2]
return [Trade(t, 1) for t, _ in top_tickers_in_sector]
def trade27():
# Beta-Weighted Portfolio: Allocate more to stocks with higher betas (more volatile
# You'd need historical market data to calculate betas
betas = {t: random.uniform(0.5, 2) for t in tickers} # Placeholder for actual betas
total_beta = sum(betas.values())
allocations = {t: betas[t] / total_beta * 100 for t in tickers}
return [Trade(t, int(allocations[t])) for t in tickers]
def trade28():
# Diversified Portfolio: Buy a mix of stocks with low correlations to each other
correlations = np.corrcoef([prices[t] for t in tickers])
chosen_tickers = []
while len(chosen_tickers) < 5 and len(tickers) > 0:
t = random.choice(tickers)
if all(correlations[tickers.index(t)][tickers.index(c)] < 0.5 for c in chosen_tickers):
chosen_tickers.append(t)
tickers.remove(t)
return [Trade(t, random.randrange(1, 3)) for t in chosen_tickers]
def trade29():
# Cointegration: Find a pair of stocks that are cointegrated and trade their spread
# This requires more complex analysis (e.g., using the Johansen test)
# For simplicity, we'll just pick a random pair and assume cointegration
i, j = random.sample(range(len(tickers)), 2)
spread = prices[tickers[i]][0] - prices[tickers[j]][0]
if spread > 0:
return [Trade(tickers[i], -1), Trade(tickers[j], 1)]
else:
return [Trade(tickers[i], 1), Trade(tickers[j], -1)]
def trade30():
# Basket Trading: Buy or sell a basket of stocks based on their correlation to a benchmark
# You'd need a benchmark ticker and its historical prices
benchmark = "SPY"
correlations = np.corrcoef([prices[t] for t in tickers + [benchmark]])[:-1, -1] # Correlate each stock with the benchmark
if np.mean(correlations) > 0.5:
return [Trade(t, 1) for t in tickers]
else:
return [Trade(t, -1) for t in tickers]
def trade31():
# Double Bottom: Buy when a stock forms a double bottom pattern
ticker = random.choice(tickers)
if prices[ticker][0] < prices[ticker][2] < prices[ticker][4] and prices[ticker][1] > prices[ticker][3]:
return [Trade(ticker, 1)]
else:
return []
def trade32():
# Double Top: Sell when a stock forms a double top pattern
ticker = random.choice(tickers)
if prices[ticker][0] > prices[ticker][2] > prices[ticker][4] and prices[ticker][1] < prices[ticker][3]:
return [Trade(ticker, -1)]
else:
return []
def trade33():
# Head and Shoulders: Sell when a stock forms a head and shoulders pattern
ticker = random.choice(tickers)
if prices[ticker][0] < prices[ticker][2] < prices[ticker][4] and prices[ticker][1] > prices[ticker][3] > prices[ticker][5]:
return [Trade(ticker, -1)]
else:
return []
def trade34
# Inverse Head and Shoulders: Buy when a stock forms an inverse head and shoulders pattern
ticker = random.choice(tickers)
if prices[ticker][0] > prices[ticker][2] > prices[ticker][4] and prices[ticker][1] < prices[ticker][3] < prices[ticker][5]:
return [Trade(ticker, 1)]
else:
return []
def trade35():
# Ascending Triangle: Buy when a stock forms an ascending triangle pattern
ticker = random.choice(tickers)
# Simplified logic: check for higher lows and flat highs
if prices[ticker][0] > prices[ticker][2] > prices[ticker][4] and prices[ticker][1] == prices[ticker][3] == prices[ticker][5]:
return [Trade(ticker, 1)]
else:
return []
def trade36():
# Descending Triangle: Sell when a stock forms a descending triangle pattern
ticker = random.choice(tickers)
# Simplified logic: check for lower highs and flat lows
if prices[ticker][0] < prices[ticker][2] < prices[ticker][4] and prices[ticker][1] == prices[ticker][3] == prices[ticker][5]:
return [Trade(ticker, -1)]
else:
return []
def trade37():
# Flag/Pennant: Buy or sell based on the direction of the flag/pennant pattern
ticker = random.choice(tickers)
# Simplified logic: check for a consolidation period after a strong move
if abs(prices[ticker][0] - np.mean(prices[ticker][1:5])) < 0.05 and abs(prices[ticker][5] - prices[ticker][6]) > 0.1:
# Buy if the prior move was up, sell if down
return [Trade(ticker, 1 if prices[ticker][5] > prices[ticker][6] else -1)]
else:
return []
def trade38():
# Gap Up: Buy when a stock opens significantly higher than its previous close
ticker = random.choice(tickers)
if prices[ticker][0] > prices[ticker][1] * 1.05: # 5% gap up
return [Trade(ticker, 1)]
else:
return []
def trade39():
# Gap Down: Sell when a stock opens significantly lower than its previous close
ticker = random.choice(tickers)
if prices[ticker][0] < prices[ticker][1] * 0.95: # 5% gap down
return [Trade(ticker, -1)]
else:
return []
def trade40():
# Rounding Bottom: Buy when a stock forms a rounding bottom pattern
ticker = random.choice(tickers)
# Simplified logic: check for a gradual price increase after a period of decline
if prices[ticker][0] > prices[ticker][2] > prices[ticker][4] and prices[ticker][1] < prices[ticker][3] < prices[ticker][5]:
return [Trade(ticker, 1)]
else:
return []
def trade41():
# Overbought/Oversold (RSI): Sell if RSI is above 70, buy if below 30
ticker = random.choice(tickers)
rsi = calculate_rsi(prices[ticker], 14) # Assuming you have an RSI calculation function
if rsi > 70:
return [Trade(ticker, -1)]
elif rsi < 30:
return [Trade(ticker, 1)]
else:
return []
def trade42():
# Bollinger Bands Breakout: Buy if price breaks above the upper band, sell if below lower
ticker = random.choice(tickers)
upper, middle, lower = calculate_bollinger_bands(prices[ticker], 20, 2) # Assuming you have a Bollinger Band calculation function
if prices[ticker][0] > upper:
return [Trade(ticker, 1)]
elif prices[ticker][0] < lower:
return [Trade(ticker, -1)]
else:
return []
def trade43():
# Channel Breakout: Buy or sell when price breaks out of a recent price channel
ticker = random.choice(tickers)
highs = [max(prices[ticker][i:i+5]) for i in range(len(prices[ticker]) - 5)]
lows = [min(prices[ticker][i:i+5]) for i in range(len(prices[ticker]) - 5)]
if prices[ticker][0] > highs[-1]:
return [Trade(ticker, 1)]
elif prices[ticker][0] < lows[-1]:
return [Trade(ticker, -1)]
else:
return []
def trade44():
# Trend Following: Buy if the 20-day moving average is rising, sell if falling
ticker = random.choice(tickers)
ma20_today = np.mean(prices[ticker][:20])
ma20_yesterday = np.mean(prices[ticker][1:21])
if ma20_today > ma20_yesterday:
return [Trade(ticker, 1)]
elif ma20_today < ma20_yesterday:
return [Trade(ticker, -1)]
else:
return []
def trade45():
# MACD Crossover: Buy when MACD line crosses above signal line, sell when below
ticker = random.choice(tickers)
macd_line, signal_line = calculate_macd(prices[ticker]) # Assuming you have a MACD calculation function
if macd_line[-1] > signal_line[-1] and macd_line[-2] <= signal_line[-2]:
return [Trade(ticker, 1)]
elif macd_line[-1] < signal_line[-1] and macd_line[-2] >= signal_line[-2]:
return [Trade(ticker, -1)]
else:
return []
def trade46():
# Stochastic Oscillator: Buy if %K crosses above %D in oversold zone, sell if opposite
ticker = random.choice(tickers)
k_line, d_line = calculate_stochastic(prices[ticker]) # Assuming you have a Stochastic calculation function
if k_line[-1] > d_line[-1] and k_line[-1] < 20:
return [Trade(ticker, 1)]
elif k_line[-1] < d_line[-1] and k_line[-1] > 80:
return [Trade(ticker, -1)]
else:
return []
def trade47():
# Volume Spike: Buy if today's volume is much higher than the average
# You'd need volume data for this strategy
ticker = random.choice(tickers)
avg_volume = np.mean(volumes[ticker][1:]) # Assuming you have 'volumes' data
if volumes[ticker][0] > avg_volume * 2:
return [Trade(ticker, 1)]
else:
return []
def trade48():
# Price Spike: Buy if today's price increase is much higher than average daily change
ticker = random.choice(tickers)
daily_changes = [(prices[ticker][i] - prices[ticker][i + 1]) / prices[ticker][i + 1] for i in range(len(prices[ticker]) - 1)]
avg_change = np.mean(daily_changes)
today_change = (prices[ticker][0] - prices[ticker][1]) / prices[ticker][1]
if today_change > avg_change * 2:
return [Trade(ticker, 1)]
else:
return []
def trade49():
# Mean Reversion (Long-term): Buy if the price is below its 200-day moving average
ticker = random.choice(tickers)
ma200 = np.mean(prices[ticker])
if prices[ticker][0] < ma200:
return [Trade(ticker, 1)]
else:
return []
def trade50():
# Trend Reversal (Parabolic SAR): Buy or sell based on the Parabolic SAR indicator
# Assuming you have a Parabolic SAR calculation function
ticker = random.choice(tickers)
sar = calculate_parabolic_sar(prices[ticker])
if prices[ticker][0] > sar[-1]:
return [Trade(ticker, 1)]
elif prices[ticker][0] < sar[-1]:
return [Trade(ticker, -1)]
else:
return []
def trade51():
# Market Outperformance: Buy stocks whose daily returns beat the market
total_market_values = [sum(prices[t][i] for t in tickers) for i in range(len(prices[tickers[0]]))]
market_return = (total_market_values[0] - total_market_values[1]) / total_market_values[1]
outperformers = [t for t in tickers if (prices[t][0] - prices[t][1]) / prices[t][1] > market_return]
if outperformers:
ticker = random.choice(outperformers)
return [Trade(ticker, 1)]
else:
return []
def trade52():
# Market Underperformance: Short stocks whose daily returns lag the market
total_market_values = [sum(prices[t][i] for t in tickers) for i in range(len(prices[tickers[0]]))]
market_return = (total_market_values[0] - total_market_values[1]) / total_market_values[1]
underperformers = [t for t in tickers if (prices[t][0] - prices[t][1]) / prices[t][1] < market_return]
if underperformers:
ticker = random.choice(underperformers)
return [Trade(ticker, -1)]
else:
return []
def trade53():
# Relative Strength to Market: Buy the stock with the highest relative strength to the market
total_market_values = [sum(prices[t][i] for t in tickers) for i in range(len(prices[tickers[0]]))]
market_return = (total_market_values[0] - total_market_values[1]) / total_market_values[1]
relative_strengths = {t: ((prices[t][0] - prices[t][1]) / prices[t][1]) - market_return for t in tickers}
ticker = max(relative_strengths, key=relative_strengths.get)
return [Trade(ticker, 1)]
def trade54():
# Relative Weakness to Market: Short the stock with the lowest relative strength to the market
total_market_values = [sum(prices[t][i] for t in tickers) for i in range(len(prices[tickers[0]]))]
market_return = (total_market_values[0] - total_market_values[1]) / total_market_values[1]
relative_strengths = {t: ((prices[t][0] - prices[t][1]) / prices[t][1]) - market_return for t in tickers}
ticker = min(relative_strengths, key=relative_strengths.get)
return [Trade(ticker, -1)]
def trade55():
# Sector vs. Market: Buy top stock from sector outperforming the market, short from underperforming
# Assuming you have sector data (e.g., 'sector_map' dict: ticker -> sector)
total_market_values = [sum(prices[t][i] for t in tickers) for i in range(len(prices[tickers[0]]))]
market_return = (total_market_values[0] - total_market_values[1]) / total_market_values[1]
sector_returns = {s: np.mean([(prices[t][0] - prices[t][1]) / prices[t][1] for t in tickers if sector_map[t] == s]) for s in set(sector_map.values())}
outperforming_sectors = [s for s in sector_returns if sector_returns[s] > market_return]
underperforming_sectors = [s for s in sector_returns if sector_returns[s] < market_return]
trades = []
if outperforming_sectors:
top_ticker = max([(t, prices[t][0]) for t in tickers if sector_map[t] == random.choice(outperforming_sectors)], key=lambda x: x[1])[0]
trades.append(Trade(top_ticker, 1))
if underperforming_sectors:
bottom_ticker = min([(t, prices[t][0]) for t in tickers if sector_map[t] == random.choice(underperforming_sectors)], key=lambda x: x[1])[0]
trades.append(Trade(bottom_ticker, -1))
return trades
def trade56():
# Market-Neutral Pairs: Long/short pairs of stocks with similar market betas
betas = {t: random.uniform(0.8, 1.2) for t in tickers} # Placeholder, calculate actual betas
pairs = [(t1, t2) for t1 in tickers for t2 in tickers if abs(betas[t1] - betas[t2]) < 0.1 and t1 != t2]
if pairs:
t1, t2 = random.choice(pairs)
return [Trade(t1, 1), Trade(t2, -1)]
else:
return []
def trade57():
# Beta Rotation: Buy high-beta stocks if the market is rising, low-beta if falling
total_market_values = [sum(prices[t][i] for t in tickers) for i in range(len(prices[tickers[0]]))]
market_return = (total_market_values[0] - total_market_values[1]) / total_market_values[1]
betas = {t: random.uniform(0.5, 2) for t in tickers} # Placeholder, calculate actual betas
if market_return > 0: # Market is rising
target_beta = 1.5 # Example target for high-beta
else:
target_beta = 0.8 # Example target for low-beta
closest_ticker = min(tickers, key=lambda t: abs(betas[t] - target_beta))
return [Trade(closest_ticker, 1 if market_return > 0 else -1)] # Buy if rising, short if falling
def trade58():
# Market Timing with Relative Strength: Buy strong stocks in up markets, sell in down markets
total_market_values = [sum(prices[t][i] for t in tickers) for i in range(len(prices[tickers[0]]))]
market_return = (total_market_values[0] - total_market_values[1]) / total_market_values[1]
relative_strengths = {t: ((prices[t][0] - prices[t][-1]) / prices[t][-1]) for t in tickers} # Calculate over a longer period (e.g., 20 days)
if market_return > 0:
strongest = max(relative_strengths, key=relative_strengths.get)
return [Trade(strongest, 1)]
else:
weakest = min(relative_strengths, key=relative_strengths.get)
return [Trade(weakest, -1)]
def trade59():
# Relative Value to Market: Buy stocks trading below their historical average relative to the market
# Requires historical data to calculate averages
total_market_values = [sum(prices[t][i] for t in tickers) for i in range(len(prices[tickers[0]]))]
relative_values = {t: prices[t][0] / total_market_values[0] for t in tickers} # Current relative value
historical_averages = {t: 0.05 for t in tickers} # Placeholder, calculate actual averages
undervalued = [t for t in tickers if relative_values[t] < historical_averages[t] * 0.95] # Allow some buffer
if undervalued:
ticker = random.choice(undervalued)
return [Trade(ticker, 1)]
else:
return []
def trade60():
# Market-Cap Weighted: Allocate trade amounts proportional to each stock's market cap relative to total market
total_market_value = sum(prices[t][0] for t in tickers)
market_caps = {t: prices[t][0] * 1000 for t in tickers} # Assuming 1000 shares outstanding for each stock
weights = {t: market_caps[t] / total_market_value for t in tickers}
total_trade_amount = 100 # Example
trades = [Trade(t, int(weights[t] * total_trade_amount)) for t in tickers]
return trades

884
extras/trading/trades_gpt-4o.py

@ -0,0 +1,884 @@
# tickers is a list of stock tickers
import tickers
# prices is a dict; the key is a ticker and the value is a list of historic prices, today first
import prices
# Trade represents a decision to buy or sell a quantity of a ticker
import Trade
import random
import numpy as np
def trade2():
# Buy top performing stock in the last 5 days
avg_prices = {ticker: np.mean(prices[ticker][:5]) for ticker in tickers}
best_ticker = max(avg_prices, key=avg_prices.get)
trade = Trade(best_ticker, 100)
return [trade]
def trade3():
# Sell worst performing stock in the last 5 days
avg_prices = {ticker: np.mean(prices[ticker][:5]) for ticker in tickers}
worst_ticker = min(avg_prices, key=avg_prices.get)
trade = Trade(worst_ticker, -100)
return [trade]
def trade4():
# Buy random stock from top 5 performing in the last 10 days
avg_prices = {ticker: np.mean(prices[ticker][:10]) for ticker in tickers}
top_5_tickers = sorted(avg_prices, key=avg_prices.get, reverse=True)[:5]
ticker = random.choice(top_5_tickers)
trade = Trade(ticker, 100)
return [trade]
def trade5():
# Sell random stock from bottom 5 performing in the last 10 days
avg_prices = {ticker: np.mean(prices[ticker][:10]) for ticker in tickers}
bottom_5_tickers = sorted(avg_prices, key=avg_prices.get)[:5]
ticker = random.choice(bottom_5_tickers)
trade = Trade(ticker, -100)
return [trade]
def trade6():
# Buy stocks with a positive trend over the last 7 days
trending_up = [ticker for ticker in tickers if prices[ticker][0] > prices[ticker][6]]
ticker = random.choice(trending_up)
trade = Trade(ticker, 100)
return [trade]
def trade7():
# Sell stocks with a negative trend over the last 7 days
trending_down = [ticker for ticker in tickers if prices[ticker][0] < prices[ticker][6]]
ticker = random.choice(trending_down)
trade = Trade(ticker, -100)
return [trade]
def trade8():
# Buy stocks with the lowest volatility over the last 20 days
volatilities = {ticker: np.std(prices[ticker][:20]) for ticker in tickers}
least_volatile = min(volatilities, key=volatilities.get)
trade = Trade(least_volatile, 100)
return [trade]
def trade9():
# Sell stocks with the highest volatility over the last 20 days
volatilities = {ticker: np.std(prices[ticker][:20]) for ticker in tickers}
most_volatile = max(volatilities, key=volatilities.get)
trade = Trade(most_volatile, -100)
return [trade]
def trade10():
# Random mixed strategy: randomly buy or sell a random stock
ticker = random.choice(tickers)
quantity = random.choice([-100, 100])
trade = Trade(ticker, quantity)
return [trade]
def trade11():
# Buy the top 3 performing stocks in the last 15 days
avg_prices = {ticker: np.mean(prices[ticker][:15]) for ticker in tickers}
top_3_tickers = sorted(avg_prices, key=avg_prices.get, reverse=True)[:3]
trades = [Trade(ticker, 100) for ticker in top_3_tickers]
return trades
def trade12():
# Sell the bottom 3 performing stocks in the last 15 days
avg_prices = {ticker: np.mean(prices[ticker][:15]) for ticker in tickers}
bottom_3_tickers = sorted(avg_prices, key=avg_prices.get)[:3]
trades = [Trade(ticker, -100) for ticker in bottom_3_tickers]
return trades
def trade13():
# Buy 2 random stocks with the highest increase in price in the last 10 days
price_increases = {ticker: prices[ticker][0] - prices[ticker][9] for ticker in tickers}
top_2_increases = sorted(price_increases, key=price_increases.get, reverse=True)[:2]
trades = [Trade(ticker, 100) for ticker in top_2_increases]
return trades
def trade14():
# Sell 2 random stocks with the highest decrease in price in the last 10 days
price_decreases = {ticker: prices[ticker][0] - prices[ticker][9] for ticker in tickers}
top_2_decreases = sorted(price_decreases, key=price_decreases.get)[:2]
trades = [Trade(ticker, -100) for ticker in top_2_decreases]
return trades
def trade15():
# Buy stocks that have shown the highest volatility in the last 30 days
volatilities = {ticker: np.std(prices[ticker][:30]) for ticker in tickers}
high_volatility_tickers = sorted(volatilities, key=volatilities.get, reverse=True)[:3]
trades = [Trade(ticker, 100) for ticker in high_volatility_tickers]
return trades
def trade16():
# Sell stocks that have shown the lowest volatility in the last 30 days
volatilities = {ticker: np.std(prices[ticker][:30]) for ticker in tickers}
low_volatility_tickers = sorted(volatilities, key=volatilities.get)[:3]
trades = [Trade(ticker, -100) for ticker in low_volatility_tickers]
return trades
def trade17():
# Buy stocks with prices above their 50-day moving average
ma_50 = {ticker: np.mean(prices[ticker][:50]) for ticker in tickers}
above_ma_tickers = [ticker for ticker in tickers if prices[ticker][0] > ma_50[ticker]]
trades = [Trade(ticker, 100) for ticker in random.sample(above_ma_tickers, min(3, len(above_ma_tickers)))]
return trades
def trade18():
# Sell stocks with prices below their 50-day moving average
ma_50 = {ticker: np.mean(prices[ticker][:50]) for ticker in tickers}
below_ma_tickers = [ticker for ticker in tickers if prices[ticker][0] < ma_50[ticker]]
trades = [Trade(ticker, -100) for ticker in random.sample(below_ma_tickers, min(3, len(below_ma_tickers)))]
return trades
def trade19():
# Mixed strategy: buy 2 random stocks and sell 2 random stocks
buy_tickers = random.sample(tickers, 2)
sell_tickers = random.sample([ticker for ticker in tickers if ticker not in buy_tickers], 2)
trades = [Trade(ticker, 100) for ticker in buy_tickers] + [Trade(ticker, -100) for ticker in sell_tickers]
return trades
def trade20():
# Buy stocks that have positive return in the last 20 days and sell those with negative return
returns = {ticker: (prices[ticker][0] - prices[ticker][19]) / prices[ticker][19] for ticker in tickers}
buy_tickers = [ticker for ticker in tickers if returns[ticker] > 0]
sell_tickers = [ticker for ticker in tickers if returns[ticker] < 0]
trades = [Trade(ticker, 100) for ticker in random.sample(buy_tickers, min(2, len(buy_tickers)))] + \
[Trade(ticker, -100) for ticker in random.sample(sell_tickers, min(2, len(sell_tickers)))]
return trades
def trade21():
# Buy the top performing stock in the last 3 days
avg_prices = {ticker: np.mean(prices[ticker][:3]) for ticker in tickers}
best_ticker = max(avg_prices, key=avg_prices.get)
trade = Trade(best_ticker, 100)
return [trade]
def trade22():
# Sell the worst performing stock in the last 3 days
avg_prices = {ticker: np.mean(prices[ticker][:3]) for ticker in tickers}
worst_ticker = min(avg_prices, key=avg_prices.get)
trade = Trade(worst_ticker, -100)
return [trade]
def trade23():
# Buy stocks that have not changed price in the last 7 days
stable_tickers = [ticker for ticker in tickers if prices[ticker][0] == prices[ticker][6]]
trades = [Trade(ticker, 100) for ticker in random.sample(stable_tickers, min(3, len(stable_tickers)))]
return trades
def trade24():
# Sell stocks that have the smallest price change in the last 5 days
smallest_changes = sorted(tickers, key=lambda t: abs(prices[t][0] - prices[t][4]))[:3]
trades = [Trade(ticker, -100) for ticker in smallest_changes]
return trades
def trade25():
# Buy random stocks from the top 10 highest priced stocks
highest_priced = sorted(tickers, key=lambda t: prices[t][0], reverse=True)[:10]
ticker = random.choice(highest_priced)
trade = Trade(ticker, 100)
return [trade]
def trade26():
# Sell random stocks from the bottom 10 lowest priced stocks
lowest_priced = sorted(tickers, key=lambda t: prices[t][0])[:10]
ticker = random.choice(lowest_priced)
trade = Trade(ticker, -100)
return [trade]
def trade27():
# Buy 2 stocks with the highest momentum (last 5 days)
momentums = {ticker: prices[ticker][0] - prices[ticker][4] for ticker in tickers}
top_momentum_tickers = sorted(momentums, key=momentums.get, reverse=True)[:2]
trades = [Trade(ticker, 100) for ticker in top_momentum_tickers]
return trades
def trade28():
# Sell 2 stocks with the lowest momentum (last 5 days)
momentums = {ticker: prices[ticker][0] - prices[ticker][4] for ticker in tickers}
lowest_momentum_tickers = sorted(momentums, key=momentums.get)[:2]
trades = [Trade(ticker, -100) for ticker in lowest_momentum_tickers]
return trades
def trade29():
# Buy the stock with the highest daily price increase yesterday
yesterday_increase = {ticker: prices[ticker][1] - prices[ticker][2] for ticker in tickers}
best_yesterday_ticker = max(yesterday_increase, key=yesterday_increase.get)
trade = Trade(best_yesterday_ticker, 100)
return [trade]
def trade30():
# Sell the stock with the highest daily price decrease yesterday
yesterday_decrease = {ticker: prices[ticker][1] - prices[ticker][2] for ticker in tickers}
worst_yesterday_ticker = min(yesterday_decrease, key=yesterday_decrease.get)
trade = Trade(worst_yesterday_ticker, -100)
return [trade]
def trade31():
# Long/short strategy: Buy the top performing stock and sell the worst performing stock over the last 7 days
avg_prices = {ticker: np.mean(prices[ticker][:7]) for ticker in tickers}
best_ticker = max(avg_prices, key=avg_prices.get)
worst_ticker = min(avg_prices, key=avg_prices.get)
trades = [Trade(best_ticker, 100), Trade(worst_ticker, -100)]
return trades
def trade32():
# Buy stocks that have had a positive return in the last 5 days and sell those with a negative return
returns = {ticker: (prices[ticker][0] - prices[ticker][4]) / prices[ticker][4] for ticker in tickers}
buy_tickers = [ticker for ticker in tickers if returns[ticker] > 0]
sell_tickers = [ticker for ticker in tickers if returns[ticker] < 0]
trades = [Trade(ticker, 100) for ticker in random.sample(buy_tickers, min(2, len(buy_tickers)))] + \
[Trade(ticker, -100) for ticker in random.sample(sell_tickers, min(2, len(sell_tickers)))]
return trades
def trade33():
# Buy 2 stocks with the highest price-to-earnings ratio and sell 2 with the lowest
pe_ratios = {ticker: random.uniform(10, 30) for ticker in tickers} # Mock P/E ratios
top_pe_tickers = sorted(pe_ratios, key=pe_ratios.get, reverse=True)[:2]
low_pe_tickers = sorted(pe_ratios, key=pe_ratios.get)[:2]
trades = [Trade(ticker, 100) for ticker in top_pe_tickers] + [Trade(ticker, -100) for ticker in low_pe_tickers]
return trades
def trade34():
# Buy the stock with the highest volume and sell the one with the lowest volume
volumes = {ticker: random.randint(1000, 10000) for ticker in tickers} # Mock volumes
high_volume_ticker = max(volumes, key=volumes.get)
low_volume_ticker = min(volumes, key=volumes.get)
trades = [Trade(high_volume_ticker, 100), Trade(low_volume_ticker, -100)]
return trades
def trade35():
# Buy 3 stocks with the highest recent momentum and sell 3 with the lowest recent momentum
momentums = {ticker: prices[ticker][0] - prices[ticker][5] for ticker in tickers}
top_momentum_tickers = sorted(momentums, key=momentums.get, reverse=True)[:3]
low_momentum_tickers = sorted(momentums, key=momentums.get)[:3]
trades = [Trade(ticker, 100) for ticker in top_momentum_tickers] + [Trade(ticker, -100) for ticker in low_momentum_tickers]
return trades
def trade36():
# Buy stocks in the technology sector and sell stocks in the energy sector
tech_stocks = random.sample(tickers, 3) # Mock tech stocks
energy_stocks = random.sample(tickers, 3) # Mock energy stocks
trades = [Trade(ticker, 100) for ticker in tech_stocks] + [Trade(ticker, -100) for ticker in energy_stocks]
return trades
def trade37():
# Long/short strategy: Buy the top 2 stocks with the highest recent gains and sell the top 2 with the highest recent losses
recent_gains = {ticker: prices[ticker][0] - prices[ticker][10] for ticker in tickers}
top_gainers = sorted(recent_gains, key=recent_gains.get, reverse=True)[:2]
top_losers = sorted(recent_gains, key=recent_gains.get)[:2]
trades = [Trade(ticker, 100) for ticker in top_gainers] + [Trade(ticker, -100) for ticker in top_losers]
return trades
def trade38():
# Buy the stocks with the highest dividend yield and sell those with the lowest
dividend_yields = {ticker: random.uniform(1, 5) for ticker in tickers} # Mock dividend yields
high_yield_tickers = sorted(dividend_yields, key=dividend_yields.get, reverse=True)[:2]
low_yield_tickers = sorted(dividend_yields, key=dividend_yields.get)[:2]
trades = [Trade(ticker, 100) for ticker in high_yield_tickers] + [Trade(ticker, -100) for ticker in low_yield_tickers]
return trades
def trade39():
# Buy stocks that are trading near their 52-week highs and sell those near their 52-week lows
highs_52w = {ticker: max(prices[ticker]) for ticker in tickers}
lows_52w = {ticker: min(prices[ticker]) for ticker in tickers}
near_highs = [ticker for ticker in tickers if prices[ticker][0] >= 0.9 * highs_52w[ticker]]
near_lows = [ticker for ticker in tickers if prices[ticker][0] <= 1.1 * lows_52w[ticker]]
trades = [Trade(ticker, 100) for ticker in random.sample(near_highs, min(2, len(near_highs)))] + \
[Trade(ticker, -100) for ticker in random.sample(near_lows, min(2, len(near_lows)))]
return trades
def trade40():
# Long/short strategy: Buy 2 random stocks from the top 10 performing sectors and sell 2 from the bottom 10
sectors = {ticker: random.choice(['Tech', 'Energy', 'Health', 'Finance', 'Retail']) for ticker in tickers}
sector_performance = {sector: random.uniform(-10, 10) for sector in set(sectors.values())}
top_sectors = sorted(sector_performance, key=sector_performance.get, reverse=True)[:2]
bottom_sectors = sorted(sector_performance, key=sector_performance.get)[:2]
buy_tickers = [ticker for ticker in tickers if sectors[ticker] in top_sectors]
sell_tickers = [ticker for ticker in tickers if sectors[ticker] in bottom_sectors]
trades = [Trade(ticker, 100) for ticker in random.sample(buy_tickers, min(2, len(buy_tickers)))] + \
[Trade(ticker, -100) for ticker in random.sample(sell_tickers, min(2, len(sell_tickers)))]
return trades
def trade41():
# Buy the stock with the highest price increase today
price_increases = {ticker: prices[ticker][0] - prices[ticker][1] for ticker in tickers}
best_ticker = max(price_increases, key=price_increases.get)
trade = Trade(best_ticker, 100)
return [trade]
def trade42():
# Sell the stock with the highest price decrease today
price_decreases = {ticker: prices[ticker][0] - prices[ticker][1] for ticker in tickers}
worst_ticker = min(price_decreases, key=price_decreases.get)
trade = Trade(worst_ticker, -100)
return [trade]
def trade43():
# Buy stocks that have had a positive return in the last 3 days
returns = {ticker: (prices[ticker][0] - prices[ticker][2]) / prices[ticker][2] for ticker in tickers}
buy_tickers = [ticker for ticker in tickers if returns[ticker] > 0]
trades = [Trade(ticker, 100) for ticker in random.sample(buy_tickers, min(3, len(buy_tickers)))]
return trades
def trade44():
# Sell stocks that have had a negative return in the last 3 days
returns = {ticker: (prices[ticker][0] - prices[ticker][2]) / prices[ticker][2] for ticker in tickers}
sell_tickers = [ticker for ticker in tickers if returns[ticker] < 0]
trades = [Trade(ticker, -100) for ticker in random.sample(sell_tickers, min(3, len(sell_tickers)))]
return trades
def trade45():
# Buy the stock with the highest average return over the last 10 days
avg_returns = {ticker: np.mean([(prices[ticker][i] - prices[ticker][i+1]) / prices[ticker][i+1] for i in range(9)]) for ticker in tickers}
best_ticker = max(avg_returns, key=avg_returns.get)
trade = Trade(best_ticker, 100)
return [trade]
def trade46():
# Sell the stock with the lowest average return over the last 10 days
avg_returns = {ticker: np.mean([(prices[ticker][i] - prices[ticker][i+1]) / prices[ticker][i+1] for i in range(9)]) for ticker in tickers}
worst_ticker = min(avg_returns, key=avg_returns.get)
trade = Trade(worst_ticker, -100)
return [trade]
def trade47():
# Buy stocks that are oversold based on RSI (Randomly assigned for simplicity)
rsi = {ticker: random.uniform(0, 100) for ticker in tickers}
oversold_tickers = [ticker for ticker in tickers if rsi[ticker] < 30]
trades = [Trade(ticker, 100) for ticker in random.sample(oversold_tickers, min(3, len(oversold_tickers)))]
return trades
def trade48():
# Sell stocks that are overbought based on RSI (Randomly assigned for simplicity)
rsi = {ticker: random.uniform(0, 100) for ticker in tickers}
overbought_tickers = [ticker for ticker in tickers if rsi[ticker] > 70]
trades = [Trade(ticker, -100) for ticker in random.sample(overbought_tickers, min(3, len(overbought_tickers)))]
return trades
def trade49():
# Buy stocks with positive momentum over the last 20 days
momentums = {ticker: prices[ticker][0] - prices[ticker][19] for ticker in tickers}
positive_momentum_tickers = [ticker for ticker in momentums if momentums[ticker] > 0]
trades = [Trade(ticker, 100) for ticker in random.sample(positive_momentum_tickers, min(3, len(positive_momentum_tickers)))]
return trades
def trade50():
# Sell stocks with negative momentum over the last 20 days
momentums = {ticker: prices[ticker][0] - prices[ticker][19] for ticker in tickers}
negative_momentum_tickers = [ticker for ticker in momentums if momentums[ticker] < 0]
trades = [Trade(ticker, -100) for ticker in random.sample(negative_momentum_tickers, min(3, len(negative_momentum_tickers)))]
return trades
def trade51():
# Buy stocks that have a high positive correlation with a randomly chosen strong performer
import scipy.stats
base_ticker = random.choice(tickers)
base_prices = prices[base_ticker]
correlations = {ticker: scipy.stats.pearsonr(base_prices, prices[ticker])[0] for ticker in tickers if ticker != base_ticker}
high_corr_tickers = [ticker for ticker, corr in correlations.items() if corr > 0.8]
trades = [Trade(ticker, 100) for ticker in random.sample(high_corr_tickers, min(3, len(high_corr_tickers)))]
return trades
def trade52():
# Sell stocks that have a high negative correlation with a randomly chosen weak performer
import scipy.stats
base_ticker = random.choice(tickers)
base_prices = prices[base_ticker]
correlations = {ticker: scipy.stats.pearsonr(base_prices, prices[ticker])[0] for ticker in tickers if ticker != base_ticker}
low_corr_tickers = [ticker for ticker, corr in correlations.items() if corr < -0.8]
trades = [Trade(ticker, -100) for ticker in random.sample(low_corr_tickers, min(3, len(low_corr_tickers)))]
return trades
def trade53():
# Long/short strategy: Buy stocks with high positive correlation and sell stocks with high negative correlation to a strong performer
import scipy.stats
base_ticker = random.choice(tickers)
base_prices = prices[base_ticker]
correlations = {ticker: scipy.stats.pearsonr(base_prices, prices[ticker])[0] for ticker in tickers if ticker != base_ticker}
high_corr_tickers = [ticker for ticker, corr in correlations.items() if corr > 0.7]
low_corr_tickers = [ticker for ticker, corr in correlations.items() if corr < -0.7]
trades = [Trade(ticker, 100) for ticker in random.sample(high_corr_tickers, min(2, len(high_corr_tickers)))] + \
[Trade(ticker, -100) for ticker in random.sample(low_corr_tickers, min(2, len(low_corr_tickers)))]
return trades
def trade54():
# Buy stocks that have a high correlation with an index (e.g., S&P 500)
import scipy.stats
index_prices = [random.uniform(1000, 5000) for _ in range(len(prices[tickers[0]]))] # Mock index prices
correlations = {ticker: scipy.stats.pearsonr(index_prices, prices[ticker])[0] for ticker in tickers}
high_corr_tickers = [ticker for ticker, corr in correlations.items() if corr > 0.8]
trades = [Trade(ticker, 100) for ticker in random.sample(high_corr_tickers, min(3, len(high_corr_tickers)))]
return trades
def trade55():
# Sell stocks that have a low correlation with an index (e.g., S&P 500)
import scipy.stats
index_prices = [random.uniform(1000, 5000) for _ in range(len(prices[tickers[0]]))] # Mock index prices
correlations = {ticker: scipy.stats.pearsonr(index_prices, prices[ticker])[0] for ticker in tickers}
low_corr_tickers = [ticker for ticker, corr in correlations.items() if corr < 0.2]
trades = [Trade(ticker, -100) for ticker in random.sample(low_corr_tickers, min(3, len(low_corr_tickers)))]
return trades
def trade56():
# Long/short strategy: Buy stocks with high correlation and sell stocks with low correlation to a randomly chosen strong performer
import scipy.stats
base_ticker = random.choice(tickers)
base_prices = prices[base_ticker]
correlations = {ticker: scipy.stats.pearsonr(base_prices, prices[ticker])[0] for ticker in tickers if ticker != base_ticker}
high_corr_tickers = [ticker for ticker, corr in correlations.items() if corr > 0.7]
low_corr_tickers = [ticker for ticker, corr in correlations.items() if corr < 0.2]
trades = [Trade(ticker, 100) for ticker in random.sample(high_corr_tickers, min(2, len(high_corr_tickers)))] + \
[Trade(ticker, -100) for ticker in random.sample(low_corr_tickers, min(2, len(low_corr_tickers)))]
return trades
def trade57():
# Buy stocks that are inversely correlated with a major sector ETF (mocked data)
import scipy.stats
sector_etf_prices = [random.uniform(50, 150) for _ in range(len(prices[tickers[0]]))] # Mock sector ETF prices
correlations = {ticker: scipy.stats.pearsonr(sector_etf_prices, prices[ticker])[0] for ticker in tickers}
inverse_corr_tickers = [ticker for ticker, corr in correlations.items() if corr < -0.7]
trades = [Trade(ticker, 100) for ticker in random.sample(inverse_corr_tickers, min(3, len(inverse_corr_tickers)))]
return trades
def trade58():
# Sell stocks that are highly correlated with a volatile index
import scipy.stats
volatile_index_prices = [random.uniform(1000, 2000) for _ in range(len(prices[tickers[0]]))] # Mock volatile index prices
correlations = {ticker: scipy.stats.pearsonr(volatile_index_prices, prices[ticker])[0] for ticker in tickers}
high_corr_tickers = [ticker for ticker, corr in correlations.items() if corr > 0.8]
trades = [Trade(ticker, -100) for ticker in random.sample(high_corr_tickers, min(3, len(high_corr_tickers)))]
return trades
def trade59():
# Buy stocks that are less correlated with the overall market (S&P 500)
import scipy.stats
market_prices = [random.uniform(1000, 5000) for _ in range(len(prices[tickers[0]]))] # Mock market index prices
correlations = {ticker: scipy.stats.pearsonr(market_prices, prices[ticker])[0] for ticker in tickers}
low_corr_tickers = [ticker for ticker, corr in correlations.items() if corr < 0.3]
trades = [Trade(ticker, 100) for ticker in random.sample(low_corr_tickers, min(3, len(low_corr_tickers)))]
return trades
def trade60():
# Sell stocks that are highly correlated with a specific commodity price (e.g., oil)
import scipy.stats
commodity_prices = [random.uniform(50, 100) for _ in range(len(prices[tickers[0]]))] # Mock commodity prices
correlations = {ticker: scipy.stats.pearsonr(commodity_prices, prices[ticker])[0] for ticker in tickers}
high_corr_tickers = [ticker for ticker, corr in correlations.items() if corr > 0.7]
trades = [Trade(ticker, -100) for ticker in random.sample(high_corr_tickers, min(3, len(high_corr_tickers)))]
return trades
def trade61():
# Buy stocks forming a "double bottom" pattern (last 5 days)
double_bottom_tickers = [ticker for ticker in tickers if prices[ticker][4] < prices[ticker][2] == prices[ticker][0] < prices[ticker][1] and prices[ticker][3] > prices[ticker][2]]
trades = [Trade(ticker, 100) for ticker in random.sample(double_bottom_tickers, min(3, len(double_bottom_tickers)))]
return trades
def trade62():
# Sell stocks forming a "double top" pattern (last 5 days)
double_top_tickers = [ticker for ticker in tickers if prices[ticker][4] > prices[ticker][2] == prices[ticker][0] > prices[ticker][1] and prices[ticker][3] < prices[ticker][2]]
trades = [Trade(ticker, -100) for ticker in random.sample(double_top_tickers, min(3, len(double_top_tickers)))]
return trades
def trade63():
# Buy stocks showing a "head and shoulders" bottom pattern (last 7 days)
hs_bottom_tickers = [ticker for ticker in tickers if prices[ticker][6] > prices[ticker][5] < prices[ticker][4] > prices[ticker][3] < prices[ticker][2] and prices[ticker][1] < prices[ticker][0]]
trades = [Trade(ticker, 100) for ticker in random.sample(hs_bottom_tickers, min(3, len(hs_bottom_tickers)))]
return trades
def trade64():
# Sell stocks showing a "head and shoulders" top pattern (last 7 days)
hs_top_tickers = [ticker for ticker in tickers if prices[ticker][6] < prices[ticker][5] > prices[ticker][4] < prices[ticker][3] > prices[ticker][2] and prices[ticker][1] > prices[ticker][0]]
trades = [Trade(ticker, -100) for ticker in random.sample(hs_top_tickers, min(3, len(hs_top_tickers)))]
return trades
def trade65():
# Buy stocks forming a "bullish flag" pattern (last 10 days)
bullish_flag_tickers = [ticker for ticker in tickers if prices[ticker][9] < prices[ticker][8] and all(prices[ticker][i] < prices[ticker][i+1] for i in range(8, 4, -1)) and all(prices[ticker][i] > prices[ticker][i+1] for i in range(4, 0, -1))]
trades = [Trade(ticker, 100) for ticker in random.sample(bullish_flag_tickers, min(3, len(bullish_flag_tickers)))]
return trades
def trade66():
# Sell stocks forming a "bearish flag" pattern (last 10 days)
bearish_flag_tickers = [ticker for ticker in tickers if prices[ticker][9] > prices[ticker][8] and all(prices[ticker][i] > prices[ticker][i+1] for i in range(8, 4, -1)) and all(prices[ticker][i] < prices[ticker][i+1] for i in range(4, 0, -1))]
trades = [Trade(ticker, -100) for ticker in random.sample(bearish_flag_tickers, min(3, len(bearish_flag_tickers)))]
return trades
def trade67():
# Buy stocks forming a "ascending triangle" pattern (last 15 days)
ascending_triangle_tickers = [ticker for ticker in tickers if prices[ticker][14] < prices[ticker][13] and prices[ticker][0] > prices[ticker][7] and all(prices[ticker][i] <= prices[ticker][i+1] for i in range(13))]
trades = [Trade(ticker, 100) for ticker in random.sample(ascending_triangle_tickers, min(3, len(ascending_triangle_tickers)))]
return trades
def trade68():
# Sell stocks forming a "descending triangle" pattern (last 15 days)
descending_triangle_tickers = [ticker for ticker in tickers if prices[ticker][14] > prices[ticker][13] and prices[ticker][0] < prices[ticker][7] and all(prices[ticker][i] >= prices[ticker][i+1] for i in range(13))]
trades = [Trade(ticker, -100) for ticker in random.sample(descending_triangle_tickers, min(3, len(descending_triangle_tickers)))]
return trades
def trade69():
# Buy stocks forming a "rounding bottom" pattern (last 20 days)
rounding_bottom_tickers = [ticker for ticker in tickers if all(prices[ticker][i] >= prices[ticker][i+1] for i in range(10)) and all(prices[ticker][i] <= prices[ticker][i+1] for i in range(10, 19))]
trades = [Trade(ticker, 100) for ticker in random.sample(rounding_bottom_tickers, min(3, len(rounding_bottom_tickers)))]
return trades
def trade70():
# Sell stocks forming a "rounding top" pattern (last 20 days)
rounding_top_tickers = [ticker for ticker in tickers if all(prices[ticker][i] <= prices[ticker][i+1] for i in range(10)) and all(prices[ticker][i] >= prices[ticker][i+1] for i in range(10, 19))]
trades = [Trade(ticker, -100) for ticker in random.sample(rounding_top_tickers, min(3, len(rounding_top_tickers)))]
return trades
def trade71():
# Buy stocks showing a strong upward trend over the last 10 days
upward_trend_tickers = [ticker for ticker in tickers if prices[ticker][0] > prices[ticker][9] and all(prices[ticker][i] >= prices[ticker][i+1] for i in range(9))]
trades = [Trade(ticker, 100) for ticker in random.sample(upward_trend_tickers, min(3, len(upward_trend_tickers)))]
return trades
def trade72():
# Sell stocks showing a strong downward trend over the last 10 days
downward_trend_tickers = [ticker for ticker in tickers if prices[ticker][0] < prices[ticker][9] and all(prices[ticker][i] <= prices[ticker][i+1] for i in range(9))]
trades = [Trade(ticker, -100) for ticker in random.sample(downward_trend_tickers, min(3, len(downward_trend_tickers)))]
return trades
def trade73():
# Buy stocks that have reverted to their mean price over the last 20 days
mean_reversion_tickers = [ticker for ticker in tickers if abs(prices[ticker][0] - np.mean(prices[ticker][:20])) < np.std(prices[ticker][:20])]
trades = [Trade(ticker, 100) for ticker in random.sample(mean_reversion_tickers, min(3, len(mean_reversion_tickers)))]
return trades
def trade74():
# Sell stocks that have deviated significantly from their mean price over the last 20 days
mean_deviation_tickers = [ticker for ticker in tickers if abs(prices[ticker][0] - np.mean(prices[ticker][:20])) > 2 * np.std(prices[ticker][:20])]
trades = [Trade(ticker, -100) for ticker in random.sample(mean_deviation_tickers, min(3, len(mean_deviation_tickers)))]
return trades
def trade75():
# Buy stocks that have shown increased volatility in the last 10 days compared to the previous 20 days
increased_volatility_tickers = [ticker for ticker in tickers if np.std(prices[ticker][:10]) > 1.5 * np.std(prices[ticker][10:30])]
trades = [Trade(ticker, 100) for ticker in random.sample(increased_volatility_tickers, min(3, len(increased_volatility_tickers)))]
return trades
def trade76():
# Sell stocks that have shown decreased volatility in the last 10 days compared to the previous 20 days
decreased_volatility_tickers = [ticker for ticker in tickers if np.std(prices[ticker][:10]) < 0.5 * np.std(prices[ticker][10:30])]
trades = [Trade(ticker, -100) for ticker in random.sample(decreased_volatility_tickers, min(3, len(decreased_volatility_tickers)))]
return trades
def trade77():
# Buy stocks that have broken above their previous 50-day high
previous_50_day_highs = {ticker: max(prices[ticker][1:51]) for ticker in tickers}
breakout_tickers = [ticker for ticker in tickers if prices[ticker][0] > previous_50_day_highs[ticker]]
trades = [Trade(ticker, 100) for ticker in random.sample(breakout_tickers, min(3, len(breakout_tickers)))]
return trades
def trade78():
# Sell stocks that have broken below their previous 50-day low
previous_50_day_lows = {ticker: min(prices[ticker][1:51]) for ticker in tickers}
breakdown_tickers = [ticker for ticker in tickers if prices[ticker][0] < previous_50_day_lows[ticker]]
trades = [Trade(ticker, -100) for ticker in random.sample(breakdown_tickers, min(3, len(breakdown_tickers)))]
return trades
def trade79():
# Buy stocks that have shown a significant upward price spike in the last 3 days
price_spike_tickers = [ticker for ticker in tickers if (prices[ticker][0] - prices[ticker][2]) / prices[ticker][2] > 0.1]
trades = [Trade(ticker, 100) for ticker in random.sample(price_spike_tickers, min(3, len(price_spike_tickers)))]
return trades
def trade80():
# Sell stocks that have shown a significant downward price spike in the last 3 days
price_drop_tickers = [ticker for ticker in tickers if (prices[ticker][0] - prices[ticker][2]) / prices[ticker][2] < -0.1]
trades = [Trade(ticker, -100) for ticker in random.sample(price_drop_tickers, min(3, len(price_drop_tickers)))]
return trades
def trade81():
# Buy stocks that have formed a "golden cross" (50-day MA crosses above 200-day MA)
golden_cross_tickers = [ticker for ticker in tickers if np.mean(prices[ticker][:50]) > np.mean(prices[ticker][:200])]
trades = [Trade(ticker, 100) for ticker in random.sample(golden_cross_tickers, min(3, len(golden_cross_tickers)))]
return trades
def trade82():
# Sell stocks that have formed a "death cross" (50-day MA crosses below 200-day MA)
death_cross_tickers = [ticker for ticker in tickers if np.mean(prices[ticker][:50]) < np.mean(prices[ticker][:200])]
trades = [Trade(ticker, -100) for ticker in random.sample(death_cross_tickers, min(3, len(death_cross_tickers)))]
return trades
def trade83():
# Buy stocks that have shown an increase in trading volume in the last 5 days
volume_increase_tickers = [ticker for ticker in tickers if np.mean(prices[ticker][:5]) > 1.2 * np.mean(prices[ticker][5:10])]
trades = [Trade(ticker, 100) for ticker in random.sample(volume_increase_tickers, min(3, len(volume_increase_tickers)))]
return trades
def trade84():
# Sell stocks that have shown a decrease in trading volume in the last 5 days
volume_decrease_tickers = [ticker for ticker in tickers if np.mean(prices[ticker][:5]) < 0.8 * np.mean(prices[ticker][5:10])]
trades = [Trade(ticker, -100) for ticker in random.sample(volume_decrease_tickers, min(3, len(volume_decrease_tickers)))]
return trades
def trade85():
# Buy stocks that have shown consistent daily gains for the last 5 days
consistent_gainers = [ticker for ticker in tickers if all(prices[ticker][i] > prices[ticker][i+1] for i in range(5))]
trades = [Trade(ticker, 100) for ticker in random.sample(consistent_gainers, min(3, len(consistent_gainers)))]
return trades
def trade86():
# Sell stocks that have shown consistent daily losses for the last 5 days
consistent_losers = [ticker for ticker in tickers if all(prices[ticker][i] < prices[ticker][i+1] for i in range(5))]
trades = [Trade(ticker, -100) for ticker in random.sample(consistent_losers, min(3, len(consistent_losers)))]
return trades
def trade87():
# Buy stocks that are trading near their all-time highs
all_time_high_tickers = [ticker for ticker in tickers if prices[ticker][0] >= 0.95 * max(prices[ticker])]
trades = [Trade(ticker, 100) for ticker in random.sample(all_time_high_tickers, min(3, len(all_time_high_tickers)))]
return trades
def trade88():
# Sell stocks that are trading near their all-time lows
all_time_low_tickers = [ticker for ticker in tickers if prices[ticker][0] <= 1.05 * min(prices[ticker])]
trades = [Trade(ticker, -100) for ticker in random.sample(all_time_low_tickers, min(3, len(all_time_low_tickers)))]
return trades
def trade89():
# Buy stocks that have gapped up at market open today
gap_up_tickers = [ticker for ticker in tickers if prices[ticker][0] > 1.05 * prices[ticker][1]]
trades = [Trade(ticker, 100) for ticker in random.sample(gap_up_tickers, min(3, len(gap_up_tickers)))]
return trades
def trade90():
# Sell stocks that have gapped down at market open today
gap_down_tickers = [ticker for ticker in tickers if prices[ticker][0] < 0.95 * prices[ticker][1]]
trades = [Trade(ticker, -100) for ticker in random.sample(gap_down_tickers, min(3, len(gap_down_tickers)))]
return trades
def trade91():
# Buy stocks that have shown a steady upward trend for the last 15 days
steady_uptrend_tickers = [ticker for ticker in tickers if all(prices[ticker][i] >= prices[ticker][i+1] for i in range(15))]
trades = [Trade(ticker, 100) for ticker in random.sample(steady_uptrend_tickers, min(3, len(steady_uptrend_tickers)))]
return trades
def trade92():
# Sell stocks that have shown a steady downward trend for the last 15 days
steady_downtrend_tickers = [ticker for ticker in tickers if all(prices[ticker][i] <= prices[ticker][i+1] for i in range(15))]
trades = [Trade(ticker, -100) for ticker in random.sample(steady_downtrend_tickers, min(3, len(steady_downtrend_tickers)))]
return trades
def trade93():
# Buy stocks that have outperformed the market index by 5% in the last 30 days
market_index_return = random.uniform(-0.05, 0.05) # Mock market index return
outperforming_tickers = [ticker for ticker in tickers if (prices[ticker][0] - prices[ticker][29]) / prices[ticker][29] > market_index_return + 0.05]
trades = [Trade(ticker, 100) for ticker in random.sample(outperforming_tickers, min(3, len(outperforming_tickers)))]
return trades
def trade94():
# Sell stocks that have underperformed the market index by 5% in the last 30 days
market_index_return = random.uniform(-0.05, 0.05) # Mock market index return
underperforming_tickers = [ticker for ticker in tickers if (prices[ticker][0] - prices[ticker][29]) / prices[ticker][29] < market_index_return - 0.05]
trades = [Trade(ticker, -100) for ticker in random.sample(underperforming_tickers, min(3, len(underperforming_tickers)))]
return trades
def trade95():
# Buy stocks that have broken above their previous 10-day high
previous_10_day_highs = {ticker: max(prices[ticker][1:11]) for ticker in tickers}
breakout_tickers = [ticker for ticker in tickers if prices[ticker][0] > previous_10_day_highs[ticker]]
trades = [Trade(ticker, 100) for ticker in random.sample(breakout_tickers, min(3, len(breakout_tickers)))]
return trades
def trade96():
# Sell stocks that have broken below their previous 10-day low
previous_10_day_lows = {ticker: min(prices[ticker][1:11]) for ticker in tickers}
breakdown_tickers = [ticker for ticker in tickers if prices[ticker][0] < previous_10_day_lows[ticker]]
trades = [Trade(ticker, -100) for ticker in random.sample(breakdown_tickers, min(3, len(breakdown_tickers)))]
return trades
def trade97():
# Buy stocks with a relative strength index (RSI) below 30 (oversold)
rsi = {ticker: random.uniform(0, 100) for ticker in tickers} # Mock RSI values
oversold_tickers = [ticker for ticker in tickers if rsi[ticker] < 30]
trades = [Trade(ticker, 100) for ticker in random.sample(oversold_tickers, min(3, len(oversold_tickers)))]
return trades
def trade98():
# Sell stocks with a relative strength index (RSI) above 70 (overbought)
rsi = {ticker: random.uniform(0, 100) for ticker in tickers} # Mock RSI values
overbought_tickers = [ticker for ticker in tickers if rsi[ticker] > 70]
trades = [Trade(ticker, -100) for ticker in random.sample(overbought_tickers, min(3, len(overbought_tickers)))]
return trades
def trade99():
# Buy stocks with a price-to-earnings ratio (P/E) below the industry average (mocked data)
pe_ratios = {ticker: random.uniform(10, 30) for ticker in tickers} # Mock P/E ratios
industry_average_pe = 20 # Mock industry average P/E
undervalued_tickers = [ticker for ticker in tickers if pe_ratios[ticker] < industry_average_pe]
trades = [Trade(ticker, 100) for ticker in random.sample(undervalued_tickers, min(3, len(undervalued_tickers)))]
return trades
def trade100():
# Sell stocks with a price-to-earnings ratio (P/E) above the industry average (mocked data)
pe_ratios = {ticker: random.uniform(10, 30) for ticker in tickers} # Mock P/E ratios
industry_average_pe = 20 # Mock industry average P/E
overvalued_tickers = [ticker for ticker in tickers if pe_ratios[ticker] > industry_average_pe]
trades = [Trade(ticker, -100) for ticker in random.sample(overvalued_tickers, min(3, len(overvalued_tickers)))]
return trades
def trade101():
# Buy stocks that have outperformed the market by more than 5% in the last 10 days
market_total = [sum(prices[ticker][i] for ticker in tickers) for i in range(10)]
market_return = (market_total[0] - market_total[-1]) / market_total[-1]
outperforming_tickers = [ticker for ticker in tickers if (prices[ticker][0] - prices[ticker][9]) / prices[ticker][9] > market_return + 0.05]
trades = [Trade(ticker, 100) for ticker in random.sample(outperforming_tickers, min(3, len(outperforming_tickers)))]
return trades
def trade102():
# Sell stocks that have underperformed the market by more than 5% in the last 10 days
market_total = [sum(prices[ticker][i] for ticker in tickers) for i in range(10)]
market_return = (market_total[0] - market_total[-1]) / market_total[-1]
underperforming_tickers = [ticker for ticker in tickers if (prices[ticker][0] - prices[ticker][9]) / prices[ticker][9] < market_return - 0.05]
trades = [Trade(ticker, -100) for ticker in random.sample(underperforming_tickers, min(3, len(underperforming_tickers)))]
return trades
def trade103():
# Buy stocks that have shown a positive return while the market showed a negative return over the last 5 days
market_total = [sum(prices[ticker][i] for ticker in tickers) for i in range(5)]
market_return = (market_total[0] - market_total[-1]) / market_total[-1]
positive_tickers = [ticker for ticker in tickers if (prices[ticker][0] - prices[ticker][4]) / prices[ticker][4] > 0 and market_return < 0]
trades = [Trade(ticker, 100) for ticker in random.sample(positive_tickers, min(3, len(positive_tickers)))]
return trades
def trade104():
# Sell stocks that have shown a negative return while the market showed a positive return over the last 5 days
market_total = [sum(prices[ticker][i] for ticker in tickers) for i in range(5)]
market_return = (market_total[0] - market_total[-1]) / market_total[-1]
negative_tickers = [ticker for ticker in tickers if (prices[ticker][0] - prices[ticker][4]) / prices[ticker][4] < 0 and market_return > 0]
trades = [Trade(ticker, -100) for ticker in random.sample(negative_tickers, min(3, len(negative_tickers)))]
return trades
def trade105():
# Buy stocks that have shown less volatility compared to the market over the last 20 days
market_total = [sum(prices[ticker][i] for ticker in tickers) for i in range(20)]
market_volatility = np.std(market_total)
low_volatility_tickers = [ticker for ticker in tickers if np.std(prices[ticker][:20]) < market_volatility]
trades = [Trade(ticker, 100) for ticker in random.sample(low_volatility_tickers, min(3, len(low_volatility_tickers)))]
return trades
def trade106():
# Sell stocks that have shown more volatility compared to the market over the last 20 days
market_total = [sum(prices[ticker][i] for ticker in tickers) for i in range(20)]
market_volatility = np.std(market_total)
high_volatility_tickers = [ticker for ticker in tickers if np.std(prices[ticker][:20]) > market_volatility]
trades = [Trade(ticker, -100) for ticker in random.sample(high_volatility_tickers, min(3, len(high_volatility_tickers)))]
return trades
def trade107():
# Buy stocks that have shown an increasing trend while the market showed a decreasing trend over the last 15 days
market_total = [sum(prices[ticker][i] for ticker in tickers) for i in range(15)]
market_trend = market_total[0] > market_total[-1]
increasing_tickers = [ticker for ticker in tickers if prices[ticker][0] > prices[ticker][14] and not market_trend]
trades = [Trade(ticker, 100) for ticker in random.sample(increasing_tickers, min(3, len(increasing_tickers)))]
return trades
def trade108():
# Sell stocks that have shown a decreasing trend while the market showed an increasing trend over the last 15 days
market_total = [sum(prices[ticker][i] for ticker in tickers) for i in range(15)]
market_trend = market_total[0] < market_total[-1]
decreasing_tickers = [ticker for ticker in tickers if prices[ticker][0] < prices[ticker][14] and market_trend]
trades = [Trade(ticker, -100) for ticker in random.sample(decreasing_tickers, min(3, len(decreasing_tickers)))]
return trades
def trade109():
# Buy stocks that have broken above their previous 10-day high while the market is flat
market_total = [sum(prices[ticker][i] for ticker in tickers) for i in range(10)]
market_flat = abs((market_total[0] - market_total[-1]) / market_total[-1]) < 0.01
previous_10_day_highs = {ticker: max(prices[ticker][1:11]) for ticker in tickers}
breakout_tickers = [ticker for ticker in tickers if prices[ticker][0] > previous_10_day_highs[ticker] and market_flat]
trades = [Trade(ticker, 100) for ticker in random.sample(breakout_tickers, min(3, len(breakout_tickers)))]
return trades
def trade110():
# Sell stocks that have broken below their previous 10-day low while the market is flat
market_total = [sum(prices[ticker][i] for ticker in tickers) for i in range(10)]
market_flat = abs((market_total[0] - market_total[-1]) / market_total[-1]) < 0.01
previous_10_day_lows = {ticker: min(prices[ticker][1:11]) for ticker in tickers}
breakdown_tickers = [ticker for ticker in tickers if prices[ticker][0] < previous_10_day_lows[ticker] and market_flat]
trades = [Trade(ticker, -100) for ticker in random.sample(breakdown_tickers, min(3, len(breakdown_tickers)))]
return trades
def trade111():
# Buy stocks that have shown a higher positive return compared to the market over the last 20 days
market_total = [sum(prices[ticker][i] for ticker in tickers) for i in range(20)]
market_return = (market_total[0] - market_total[-1]) / market_total[-1]
higher_positive_tickers = [ticker for ticker in tickers if (prices[ticker][0] - prices[ticker][19]) / prices[ticker][19] > market_return]
trades = [Trade(ticker, 100) for ticker in random.sample(higher_positive_tickers, min(3, len(higher_positive_tickers)))]
return trades
def trade112():
# Sell stocks that have shown a higher negative return compared to the market over the last 20 days
market_total = [sum(prices[ticker][i] for ticker in tickers) for i in range(20)]
market_return = (market_total[0] - market_total[-1]) / market_total[-1]
higher_negative_tickers = [ticker for ticker in tickers if (prices[ticker][0] - prices[ticker][19]) / prices[ticker][19] < market_return]
trades = [Trade(ticker, -100) for ticker in random.sample(higher_negative_tickers, min(3, len(higher_negative_tickers)))]
return trades
def trade113():
# Buy stocks that have shown less drawdown compared to the market over the last 30 days
market_total = [sum(prices[ticker][i] for ticker in tickers) for i in range(30)]
market_drawdown = min(market_total) / max(market_total)
less_drawdown_tickers = [ticker for ticker in tickers if min(prices[ticker][:30]) / max(prices[ticker][:30]) > market_drawdown]
trades = [Trade(ticker, 100) for ticker in random.sample(less_drawdown_tickers, min(3, len(less_drawdown_tickers)))]
return trades
def trade114():
# Sell stocks that have shown more drawdown compared to the market over the last 30 days
market_total = [sum(prices[ticker][i] for ticker in tickers) for i in range(30)]
market_drawdown = min(market_total) / max(market_total)
more_drawdown_tickers = [ticker for ticker in tickers if min(prices[ticker][:30]) / max(prices[ticker][:30]) < market_drawdown]
trades = [Trade(ticker, -100) for ticker in random.sample(more_drawdown_tickers, min(3, len(more_drawdown_tickers)))]
return trades
def trade115():
# Buy stocks that have had a smaller price range compared to the market over the last 15 days
market_total = [sum(prices[ticker][i] for ticker in tickers) for i in range(15)]
market_range = max(market_total) - min(market_total)
small_range_tickers = [ticker for ticker in tickers if max(prices[ticker][:15]) - min(prices[ticker][:15]) < market_range]
trades = [Trade(ticker, 100) for ticker in random.sample(small_range_tickers, min(3, len(small_range_tickers)))]
return trades
def trade116():
# Sell stocks that have had a larger price range compared to the market over the last 15 days
market_total = [sum(prices[ticker][i] for ticker in tickers) for i in range(15)]
market_range = max(market_total) - min(market_total)
large_range_tickers = [ticker for ticker in tickers if max(prices[ticker][:15]) - min(prices[ticker][:15]) > market_range]
trades = [Trade(ticker, -100) for ticker in random.sample(large_range_tickers, min(3, len(large_range_tickers)))]
return trades
def trade117():
# Buy stocks that have consistently stayed above their market-relative average price in the last 10 days
market_total = [sum(prices[ticker][i] for ticker in tickers) for i in range(10)]
market_avg = sum(market_total) / len(market_total)
consistent_above_avg_tickers = [ticker for ticker in tickers if all(prices[ticker][i] > market_avg for i in range(10))]
trades = [Trade(ticker, 100) for ticker in random.sample(consistent_above_avg_tickers, min(3, len(consistent_above_avg_tickers)))]
return trades
def trade118():
# Sell stocks that have consistently stayed below their market-relative average price in the last 10 days
market_total = [sum(prices[ticker][i] for ticker in tickers) for i in range(10)]
market_avg = sum(market_total) / len(market_total)
consistent_below_avg_tickers = [ticker for ticker in tickers if all(prices[ticker][i] < market_avg for i in range(10))]
trades = [Trade(ticker, -100) for ticker in random.sample(consistent_below_avg_tickers, min(3, len(consistent_below_avg_tickers)))]
return trades
def trade119():
# Buy stocks that have shown a positive correlation with the market trend over the last 20 days
market_total = [sum(prices[ticker][i] for ticker in tickers) for i in range(20)]
market_trend = scipy.stats.linregress(range(20), market_total).slope
positive_corr_tickers = [ticker for ticker in tickers if scipy.stats.pearsonr(prices[ticker][:20], market_total)[0] > 0.5]
trades = [Trade(ticker, 100) for ticker in random.sample(positive_corr_tickers, min(3, len(positive_corr_tickers)))]
return trades
def trade120():
# Sell stocks that have shown a negative correlation with the market trend over the last 20 days
market_total = [sum(prices[ticker][i] for ticker in tickers) for i in range(20)]
market_trend = scipy.stats.linregress(range(20), market_total).slope
negative_corr_tickers = [ticker for ticker in tickers if scipy.stats.pearsonr(prices[ticker][:20], market_total)[0] < -0.5]
trades = [Trade(ticker, -100) for ticker in random.sample(negative_corr_tickers, min(3, len(negative_corr_tickers)))]
return trades

BIN
important.jpg

Binary file not shown.

After

Width:  |  Height:  |  Size: 356 KiB

3
requirements.txt

@ -37,3 +37,6 @@ ollama
accelerate accelerate
sentencepiece sentencepiece
bitsandbytes bitsandbytes
psutil
setuptools
speedtest-cli

BIN
resources.jpg

Binary file not shown.

After

Width:  |  Height:  |  Size: 439 KiB

10
week1/Intermediate Python.ipynb

@ -23,7 +23,7 @@
"source": [ "source": [
"## First: if you need a refresher on the foundations\n", "## First: if you need a refresher on the foundations\n",
"\n", "\n",
"I'm going to defer to an AI friend for this, because these explanations are so well written with great examples. Copy and paste the code examples into a new cell to give them a try.\n", "I'm going to defer to an AI friend for this, because these explanations are so well written with great examples. Copy and paste the code examples into a new cell to give them a try. Pick whichever section(s) you'd like to brush up on.\n",
"\n", "\n",
"**Python imports:** \n", "**Python imports:** \n",
"https://chatgpt.com/share/672f9f31-8114-8012-be09-29ef0d0140fb\n", "https://chatgpt.com/share/672f9f31-8114-8012-be09-29ef0d0140fb\n",
@ -40,8 +40,14 @@
"**Python lists, dicts and sets**, including the `get()` method: \n", "**Python lists, dicts and sets**, including the `get()` method: \n",
"https://chatgpt.com/share/672fa225-3f04-8012-91af-f9c95287da8d\n", "https://chatgpt.com/share/672fa225-3f04-8012-91af-f9c95287da8d\n",
"\n", "\n",
"**Python files** including modes, encoding, context managers, Path, glob.glob: \n",
"https://chatgpt.com/share/673b53b2-6d5c-8012-a344-221056c2f960\n",
"\n",
"**Python classes:** \n", "**Python classes:** \n",
"https://chatgpt.com/share/672fa07a-1014-8012-b2ea-6dc679552715" "https://chatgpt.com/share/672fa07a-1014-8012-b2ea-6dc679552715\n",
"\n",
"**Pickling Python objects and converting to JSON:** \n",
"https://chatgpt.com/share/673b553e-9d0c-8012-9919-f3bb5aa23e31"
] ]
}, },
{ {

88
week1/day1.ipynb

@ -37,15 +37,34 @@
"\n", "\n",
"## More troubleshooting\n", "## More troubleshooting\n",
"\n", "\n",
"Please see the [troubleshooting](troubleshooting.ipynb) notebook in this folder to diagnose and fix common problems.\n", "Please see the [troubleshooting](troubleshooting.ipynb) notebook in this folder to diagnose and fix common problems. At the very end of it is a diagnostics script with some useful debug info.\n",
"\n", "\n",
"## If this is old hat!\n", "## If this is old hat!\n",
"\n", "\n",
"If you're already comfortable with today's material, please hang in there; you can move swiftly through the first few labs - we will get much more in depth as the weeks progress.\n", "If you're already comfortable with today's material, please hang in there; you can move swiftly through the first few labs - we will get much more in depth as the weeks progress.\n",
"\n", "\n",
"## Business value of these exercises\n", "<table style=\"margin: 0; text-align: left;\">\n",
"\n", " <tr>\n",
"A final thought. While I've designed these notebooks to be educational, I've also tried to make them enjoyable. We'll do fun things like have LLMs tell jokes and argue with each other. But fundamentally, my goal is to teach skills you can apply in business. I'll explain business implications as we go, and it's worth keeping this in mind: as you build experience with models and techniques, think of ways you could put this into action at work today. Please do contact me if you'd like to discuss more or if you have ideas to bounce off me." " <td style=\"width: 150px; height: 150px; vertical-align: middle;\">\n",
" <img src=\"../important.jpg\" width=\"150\" height=\"150\" style=\"display: block;\" />\n",
" </td>\n",
" <td>\n",
" <h2 style=\"color:#900;\">Please read - important note</h2>\n",
" <span style=\"color:#900;\">The way I collaborate with you may be different to other courses you've taken. I prefer not to type code while you watch. Rather, I execute Jupyter Labs, like this, and give you an intuition for what's going on. My suggestion is that you do this with me, either at the same time, or (perhaps better) right afterwards. Add print statements to understand what's going on, and then come up with your own variations. If you have a Github account, use this to showcase your variations. Not only is this essential practice, but it demonstrates your skills to others, including perhaps future clients or employers...</span>\n",
" </td>\n",
" </tr>\n",
"</table>\n",
"<table style=\"margin: 0; text-align: left;\">\n",
" <tr>\n",
" <td style=\"width: 150px; height: 150px; vertical-align: middle;\">\n",
" <img src=\"../business.jpg\" width=\"150\" height=\"150\" style=\"display: block;\" />\n",
" </td>\n",
" <td>\n",
" <h2 style=\"color:#181;\">Business value of these exercises</h2>\n",
" <span style=\"color:#181;\">A final thought. While I've designed these notebooks to be educational, I've also tried to make them enjoyable. We'll do fun things like have LLMs tell jokes and argue with each other. But fundamentally, my goal is to teach skills you can apply in business. I'll explain business implications as we go, and it's worth keeping this in mind: as you build experience with models and techniques, think of ways you could put this into action at work today. Please do contact me if you'd like to discuss more or if you have ideas to bounce off me.</span>\n",
" </td>\n",
" </tr>\n",
"</table>"
] ]
}, },
{ {
@ -144,9 +163,6 @@
"# If you're not familiar with Classes, check out the \"Intermediate Python\" notebook\n", "# If you're not familiar with Classes, check out the \"Intermediate Python\" notebook\n",
"\n", "\n",
"class Website:\n", "class Website:\n",
" \"\"\"\n",
" A utility class to represent a Website that we have scraped\n",
" \"\"\"\n",
"\n", "\n",
" def __init__(self, url):\n", " def __init__(self, url):\n",
" \"\"\"\n", " \"\"\"\n",
@ -179,7 +195,7 @@
} }
], ],
"source": [ "source": [
"# Let's try one out\n", "# Let's try one out. Change the website and add print statements to follow along.\n",
"\n", "\n",
"ed = Website(\"https://jaivikhimalay.com\")\n", "ed = Website(\"https://jaivikhimalay.com\")\n",
"#ed = Website(\"https://edwarddonner.com\")\n", "#ed = Website(\"https://edwarddonner.com\")\n",
@ -362,6 +378,8 @@
} }
], ],
"source": [ "source": [
"# Try this out, and then try for a few more websites\n",
"\n",
"messages_for(ed)" "messages_for(ed)"
] ]
}, },
@ -570,11 +588,59 @@
"id": "c951be1a-7f1b-448f-af1f-845978e47e2c", "id": "c951be1a-7f1b-448f-af1f-845978e47e2c",
"metadata": {}, "metadata": {},
"source": [ "source": [
"## Business Applications\n", "<table style=\"margin: 0; text-align: left;\">\n",
" <tr>\n",
" <td style=\"width: 150px; height: 150px; vertical-align: middle;\">\n",
" <img src=\"../business.jpg\" width=\"150\" height=\"150\" style=\"display: block;\" />\n",
" </td>\n",
" <td>\n",
" <h2 style=\"color:#181;\">Business applications</h2>\n",
" <span style=\"color:#181;\">In this exercise, you experienced calling the Cloud API of a Frontier Model (a leading model at the frontier of AI) for the first time. We will be using APIs like OpenAI at many stages in the course, in addition to building our own LLMs.\n",
"\n",
"More specifically, we've applied this to Summarization - a classic Gen AI use case to make a summary. This can be applied to any business vertical - summarizing the news, summarizing financial performance, summarizing a resume in a cover letter - the applications are limitless. Consider how you could apply Summarization in your business, and try prototyping a solution.</span>\n",
" </td>\n",
" </tr>\n",
"</table>\n",
"\n",
"<table style=\"margin: 0; text-align: left;\">\n",
" <tr>\n",
" <td style=\"width: 150px; height: 150px; vertical-align: middle;\">\n",
" <img src=\"../important.jpg\" width=\"150\" height=\"150\" style=\"display: block;\" />\n",
" </td>\n",
" <td>\n",
" <h2 style=\"color:#900;\">Before you continue - now try yourself</h2>\n",
" <span style=\"color:#900;\">Use the cell below to make your own simple commercial example. Stick with the summarization use case for now. Here's an idea: write something that will take the contents of an email, and will suggest an appropriate short subject line for the email. That's the kind of feature that might be built into a commercial email tool.</span>\n",
" </td>\n",
" </tr>\n",
"</table>"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "00743dac-0e70-45b7-879a-d7293a6f68a6",
"metadata": {},
"outputs": [],
"source": [
"# Step 1: Create your prompts\n",
"\n",
"system_prompt = \"something here\"\n",
"user_prompt = \"\"\"\n",
" Lots of text\n",
" Can be pasted here\n",
"\"\"\"\n",
"\n",
"# Step 2: Make the messages list\n",
"\n",
"messages = [] # fill this in\n",
"\n",
"# Step 3: Call OpenAI\n",
"\n",
"response =\n",
"\n", "\n",
"In this exercise, you experienced calling the API of a Frontier Model (a leading model at the frontier of AI) for the first time. This is broadly applicable across Gen AI use cases and we will be using APIs like OpenAI at many stages in the course, in addition to building our own LLMs.\n", "# Step 4: print the result\n",
"\n", "\n",
"More specifically, we've applied this to Summarization - a classic Gen AI use case to make a summary. This can be applied to any business vertical - summarizing the news, summarizing financial performance, summarizing a resume in a cover letter - the applications are limitless. Consider how you could apply Summarization in your business, and try prototyping a solution." "print("
] ]
}, },
{ {

44
week1/day2 EXERCISE.ipynb

@ -4,6 +4,37 @@
"cell_type": "markdown", "cell_type": "markdown",
"id": "d15d8294-3328-4e07-ad16-8a03e9bbfdb9", "id": "d15d8294-3328-4e07-ad16-8a03e9bbfdb9",
"metadata": {}, "metadata": {},
"source": [
"# Welcome to your first assignment!\n",
"\n",
"Instructions are below. Please give this a try, and look in the solutions folder if you get stuck (or feel free to ask me!)"
]
},
{
"cell_type": "markdown",
"id": "ada885d9-4d42-4d9b-97f0-74fbbbfe93a9",
"metadata": {},
"source": [
"<table style=\"margin: 0; text-align: left;\">\n",
" <tr>\n",
" <td style=\"width: 150px; height: 150px; vertical-align: middle;\">\n",
" <img src=\"../resources.jpg\" width=\"150\" height=\"150\" style=\"display: block;\" />\n",
" </td>\n",
" <td>\n",
" <h2 style=\"color:#f71;\">Just before we get to the assignment --</h2>\n",
" <span style=\"color:#f71;\">I thought I'd take a second to point you at this page of useful resources for the course. This includes links to all the slides.<br/>\n",
" <a href=\"https://edwarddonner.com/2024/11/13/llm-engineering-resources/\">https://edwarddonner.com/2024/11/13/llm-engineering-resources/</a><br/>\n",
" Please keep this bookmarked, and I'll continue to add more useful links there over time.\n",
" </span>\n",
" </td>\n",
" </tr>\n",
"</table>"
]
},
{
"cell_type": "markdown",
"id": "6e9fa1fc-eac5-4d1d-9be4-541b3f2b3458",
"metadata": {},
"source": [ "source": [
"# HOMEWORK EXERCISE ASSIGNMENT\n", "# HOMEWORK EXERCISE ASSIGNMENT\n",
"\n", "\n",
@ -29,7 +60,10 @@
"You should see the message `Ollama is running`. \n", "You should see the message `Ollama is running`. \n",
"\n", "\n",
"If not, bring up a new Terminal (Mac) or Powershell (Windows) and enter `ollama serve` \n", "If not, bring up a new Terminal (Mac) or Powershell (Windows) and enter `ollama serve` \n",
"Then try [http://localhost:11434/](http://localhost:11434/) again." "And in another Terminal (Mac) or Powershell (Windows), enter `ollama pull llama3.2` \n",
"Then try [http://localhost:11434/](http://localhost:11434/) again.\n",
"\n",
"If Ollama is slow on your machine, try using `llama3.2:1b` as an alternative. Run `ollama pull llama3.2:1b` from a Terminal or Powershell, and change the code below from `MODEL = \"llama3.2\"` to `MODEL = \"llama3.2:1b\"`"
] ]
}, },
{ {
@ -183,14 +217,6 @@
"print(response['message']['content'])" "print(response['message']['content'])"
] ]
}, },
{
"cell_type": "code",
"execution_count": null,
"id": "9a611b05-b5b0-4c83-b82d-b3a39ffb917d",
"metadata": {},
"outputs": [],
"source": []
},
{ {
"cell_type": "markdown", "cell_type": "markdown",
"id": "1622d9bb-5c68-4d4e-9ca4-b492c751f898", "id": "1622d9bb-5c68-4d4e-9ca4-b492c751f898",

62
week1/day5.ipynb

@ -52,7 +52,7 @@
"load_dotenv()\n", "load_dotenv()\n",
"api_key = os.getenv('OPENAI_API_KEY')\n", "api_key = os.getenv('OPENAI_API_KEY')\n",
"\n", "\n",
"if api_key and api_key[:8]=='sk-proj-':\n", "if api_key and api_key.startswith('sk-proj-') and len(api_key)>10:\n",
" print(\"API key looks good so far\")\n", " print(\"API key looks good so far\")\n",
"else:\n", "else:\n",
" print(\"There might be a problem with your API key? Please visit the troubleshooting notebook!\")\n", " print(\"There might be a problem with your API key? Please visit the troubleshooting notebook!\")\n",
@ -383,6 +383,8 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"# Try changing the system prompt to the humorous version when you make the Brochure for Hugging Face:\n",
"\n",
"stream_brochure(\"HuggingFace\", \"https://huggingface.co\")" "stream_brochure(\"HuggingFace\", \"https://huggingface.co\")"
] ]
}, },
@ -391,19 +393,65 @@
"id": "a27bf9e0-665f-4645-b66b-9725e2a959b5", "id": "a27bf9e0-665f-4645-b66b-9725e2a959b5",
"metadata": {}, "metadata": {},
"source": [ "source": [
"## Business Applications\n", "<table style=\"margin: 0; text-align: left;\">\n",
"\n", " <tr>\n",
"In this exercise we extended the Day 1 code to make multiple LLM calls, and generate a document.\n", " <td style=\"width: 150px; height: 150px; vertical-align: middle;\">\n",
" <img src=\"../business.jpg\" width=\"150\" height=\"150\" style=\"display: block;\" />\n",
" </td>\n",
" <td>\n",
" <h2 style=\"color:#181;\">Business applications</h2>\n",
" <span style=\"color:#181;\">In this exercise we extended the Day 1 code to make multiple LLM calls, and generate a document.\n",
"\n", "\n",
"In terms of techniques, this is perhaps the first example of Agentic AI design patterns, as we combined multiple calls to LLMs. This will feature more in Week 2, and then we will return to Agentic AI in a big way in Week 8 when we build a fully autonomous Agent solution.\n", "This is perhaps the first example of Agentic AI design patterns, as we combined multiple calls to LLMs. This will feature more in Week 2, and then we will return to Agentic AI in a big way in Week 8 when we build a fully autonomous Agent solution.\n",
"\n", "\n",
"In terms of business applications - generating content in this way is one of the very most common Use Cases. As with summarization, this can be applied to any business vertical. Write marketing content, generate a product tutorial from a spec, create personalized email content, and so much more. Explore how you can apply content generation to your business, and try making yourself a proof-of-concept prototype." "Generating content in this way is one of the very most common Use Cases. As with summarization, this can be applied to any business vertical. Write marketing content, generate a product tutorial from a spec, create personalized email content, and so much more. Explore how you can apply content generation to your business, and try making yourself a proof-of-concept prototype.</span>\n",
" </td>\n",
" </tr>\n",
"</table>"
]
},
{
"cell_type": "markdown",
"id": "14b2454b-8ef8-4b5c-b928-053a15e0d553",
"metadata": {},
"source": [
"<table style=\"margin: 0; text-align: left;\">\n",
" <tr>\n",
" <td style=\"width: 150px; height: 150px; vertical-align: middle;\">\n",
" <img src=\"../important.jpg\" width=\"150\" height=\"150\" style=\"display: block;\" />\n",
" </td>\n",
" <td>\n",
" <h2 style=\"color:#900;\">Before you move to Week 2 (which is tons of fun)</h2>\n",
" <span style=\"color:#900;\">Please see the week1 EXERCISE notebook for your challenge for the end of week 1. This will give you some essential practice working with Frontier APIs, and prepare you well for Week 2.</span>\n",
" </td>\n",
" </tr>\n",
"</table>"
]
},
{
"cell_type": "markdown",
"id": "17b64f0f-7d33-4493-985a-033d06e8db08",
"metadata": {},
"source": [
"<table style=\"margin: 0; text-align: left;\">\n",
" <tr>\n",
" <td style=\"width: 150px; height: 150px; vertical-align: middle;\">\n",
" <img src=\"../resources.jpg\" width=\"150\" height=\"150\" style=\"display: block;\" />\n",
" </td>\n",
" <td>\n",
" <h2 style=\"color:#f71;\">A reminder on 2 useful resources</h2>\n",
" <span style=\"color:#f71;\">1. The resources for the course are available <a href=\"https://edwarddonner.com/2024/11/13/llm-engineering-resources/\">here.</a><br/>\n",
" 2. I'm on LinkedIn <a href=\"https://www.linkedin.com/in/eddonner/\">here</a> and I love connecting with people taking the course!\n",
" </span>\n",
" </td>\n",
" </tr>\n",
"</table>"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": null,
"id": "22e878f1-08fe-4465-b50c-869352174eae", "id": "3de35771-455f-40b5-ba44-7c0a6b7c427a",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [] "source": []

419
week1/diagnostics.py

@ -0,0 +1,419 @@
import os
import sys
import platform
import subprocess
import shutil
import time
import ssl
import tempfile
from pathlib import Path
from datetime import datetime
class Diagnostics:
FILENAME = 'report.txt'
def __init__(self):
self.errors = []
self.warnings = []
if os.path.exists(self.FILENAME):
os.remove(self.FILENAME)
def log(self, message):
print(message)
with open(self.FILENAME, 'a', encoding='utf-8') as f:
f.write(message + "\n")
def start(self):
now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
self.log(f"Starting diagnostics at {now}\n")
def end(self):
now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
self.log(f"\n\nCompleted diagnostics at {now}\n")
print("\nPlease send these diagnostics to me at ed@edwarddonner.com")
print(f"Either copy & paste the above output into an email, or attach the file {self.FILENAME} that has been created in this directory.")
def _log_error(self, message):
self.log(f"ERROR: {message}")
self.errors.append(message)
def _log_warning(self, message):
self.log(f"WARNING: {message}")
self.warnings.append(message)
def run(self):
self.start()
self._step1_system_info()
self._step2_check_files()
self._step3_git_repo()
self._step4_check_env_file()
self._step5_anaconda_check()
self._step6_virtualenv_check()
self._step7_network_connectivity()
self._step8_environment_variables()
self._step9_additional_diagnostics()
if self.warnings:
self.log("\n===== Warnings Found =====")
self.log("The following warnings were detected. They might not prevent the program from running but could cause unexpected behavior:")
for warning in self.warnings:
self.log(f"- {warning}")
if self.errors:
self.log("\n===== Errors Found =====")
self.log("The following critical issues were detected. Please address them before proceeding:")
for error in self.errors:
self.log(f"- {error}")
if not self.errors and not self.warnings:
self.log("\n✅ All diagnostics passed successfully!")
self.end()
def _step1_system_info(self):
self.log("===== System Information =====")
try:
system = platform.system()
self.log(f"Operating System: {system}")
if system == "Windows":
release, version, csd, ptype = platform.win32_ver()
self.log(f"Windows Release: {release}")
self.log(f"Windows Version: {version}")
elif system == "Darwin":
release, version, machine = platform.mac_ver()
self.log(f"MacOS Version: {release}")
else:
self.log(f"Platform: {platform.platform()}")
self.log(f"Architecture: {platform.architecture()}")
self.log(f"Machine: {platform.machine()}")
self.log(f"Processor: {platform.processor()}")
try:
import psutil
ram = psutil.virtual_memory()
total_ram_gb = ram.total / (1024 ** 3)
available_ram_gb = ram.available / (1024 ** 3)
self.log(f"Total RAM: {total_ram_gb:.2f} GB")
self.log(f"Available RAM: {available_ram_gb:.2f} GB")
if available_ram_gb < 2:
self._log_warning(f"Low available RAM: {available_ram_gb:.2f} GB")
except ImportError:
self._log_warning("psutil module not found. Cannot determine RAM information.")
total, used, free = shutil.disk_usage(os.path.expanduser("~"))
free_gb = free / (1024 ** 3)
self.log(f"Free Disk Space: {free_gb:.2f} GB")
if free_gb < 5:
self._log_warning(f"Low disk space: {free_gb:.2f} GB free")
except Exception as e:
self._log_error(f"System information check failed: {e}")
def _step2_check_files(self):
self.log("\n===== File System Information =====")
try:
current_dir = os.getcwd()
self.log(f"Current Directory: {current_dir}")
# Check write permissions
test_file = Path(current_dir) / ".test_write_permission"
try:
test_file.touch(exist_ok=True)
test_file.unlink()
self.log("Write permission: OK")
except Exception as e:
self._log_error(f"No write permission in current directory: {e}")
self.log("\nFiles in Current Directory:")
try:
for item in sorted(os.listdir(current_dir)):
self.log(f" - {item}")
except Exception as e:
self._log_error(f"Cannot list directory contents: {e}")
except Exception as e:
self._log_error(f"File system check failed: {e}")
def _step3_git_repo(self):
self.log("\n===== Git Repository Information =====")
try:
result = subprocess.run(['git', 'rev-parse', '--show-toplevel'],
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
if result.returncode == 0:
git_root = result.stdout.strip()
self.log(f"Git Repository Root: {git_root}")
result = subprocess.run(['git', 'rev-parse', 'HEAD'],
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
if result.returncode == 0:
self.log(f"Current Commit: {result.stdout.strip()}")
else:
self._log_warning(f"Could not get current commit: {result.stderr.strip()}")
result = subprocess.run(['git', 'remote', 'get-url', 'origin'],
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
if result.returncode == 0:
self.log(f"Remote Origin: {result.stdout.strip()}")
else:
self._log_warning("No remote 'origin' configured")
else:
self._log_warning("Not a git repository")
except FileNotFoundError:
self._log_warning("Git is not installed or not in PATH")
except Exception as e:
self._log_error(f"Git check failed: {e}")
def _step4_check_env_file(self):
self.log("\n===== Environment File Check =====")
try:
result = subprocess.run(['git', 'rev-parse', '--show-toplevel'],
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
if result.returncode == 0:
git_root = result.stdout.strip()
env_path = os.path.join(git_root, '.env')
if os.path.isfile(env_path):
self.log(f".env file exists at: {env_path}")
try:
with open(env_path, 'r') as f:
has_api_key = any(line.strip().startswith('OPENAI_API_KEY=') for line in f)
if has_api_key:
self.log("OPENAI_API_KEY found in .env file")
else:
self._log_warning("OPENAI_API_KEY not found in .env file")
except Exception as e:
self._log_error(f"Cannot read .env file: {e}")
else:
self._log_warning(".env file not found in project root")
# Check for additional .env files
for root, _, files in os.walk(git_root):
if '.env' in files and os.path.join(root, '.env') != env_path:
self._log_warning(f"Additional .env file found at: {os.path.join(root, '.env')}")
else:
self._log_warning("Git root directory not found. Cannot perform .env file check.")
except FileNotFoundError:
self._log_warning("Git is not installed or not in PATH")
except Exception as e:
self._log_error(f"Environment file check failed: {e}")
def _step5_anaconda_check(self):
self.log("\n===== Anaconda Environment Check =====")
try:
conda_prefix = os.environ.get('CONDA_PREFIX')
if conda_prefix:
self.log("Anaconda environment is active:")
self.log(f"Environment Path: {conda_prefix}")
self.log(f"Environment Name: {os.path.basename(conda_prefix)}")
conda_exe = os.environ.get('CONDA_EXE', 'conda')
result = subprocess.run([conda_exe, '--version'],
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
if result.returncode == 0:
self.log(f"Conda Version: {result.stdout.strip()}")
else:
self._log_warning("Could not determine Conda version")
self._check_python_packages()
else:
self.log("No active Anaconda environment detected")
except Exception as e:
self._log_error(f"Anaconda environment check failed: {e}")
def _step6_virtualenv_check(self):
self.log("\n===== Virtualenv Check =====")
try:
virtual_env = os.environ.get('VIRTUAL_ENV')
if virtual_env:
self.log("Virtualenv is active:")
self.log(f"Environment Path: {virtual_env}")
self.log(f"Environment Name: {os.path.basename(virtual_env)}")
self._check_python_packages()
else:
self.log("No active virtualenv detected")
if not virtual_env and not os.environ.get('CONDA_PREFIX'):
self._log_warning("Neither virtualenv nor Anaconda environment is active")
except Exception as e:
self._log_error(f"Virtualenv check failed: {e}")
def _check_python_packages(self):
self.log("\nPython Environment:")
self.log(f"Python Version: {sys.version}")
self.log(f"Python Executable: {sys.executable}")
required_packages = ['openai', 'python-dotenv', 'requests', 'gradio', 'transformers']
try:
import pkg_resources
installed = {pkg.key: pkg.version for pkg in pkg_resources.working_set}
self.log("\nRequired Package Versions:")
for package in required_packages:
if package in installed:
self.log(f"{package}: {installed[package]}")
else:
self._log_error(f"Required package '{package}' is not installed")
# Check for potentially conflicting packages
problem_pairs = [
('openai', 'openai-python'),
('python-dotenv', 'dotenv')
]
for pkg1, pkg2 in problem_pairs:
if pkg1 in installed and pkg2 in installed:
self._log_warning(f"Potentially conflicting packages: {pkg1} and {pkg2}")
except ImportError:
self._log_error("Could not import 'pkg_resources' to check installed packages")
except Exception as e:
self._log_error(f"Package check failed: {e}")
def _step7_network_connectivity(self):
self.log("\n===== Network Connectivity Check =====")
try:
self.log(f"SSL Version: {ssl.OPENSSL_VERSION}")
import requests
import speedtest # Importing the speedtest-cli library
# Basic connectivity check
urls = [
'https://www.google.com',
'https://www.cloudflare.com'
]
connected = False
for url in urls:
try:
start_time = time.time()
response = requests.get(url, timeout=10)
elapsed_time = time.time() - start_time
response.raise_for_status()
self.log(f"✓ Connected to {url}")
self.log(f" Response time: {elapsed_time:.2f}s")
if elapsed_time > 2:
self._log_warning(f"Slow response from {url}: {elapsed_time:.2f}s")
connected = True
break
except requests.exceptions.RequestException as e:
self._log_warning(f"Failed to connect to {url}: {e}")
else:
self.log("Basic connectivity OK")
if not connected:
self._log_error("Failed to connect to any test URLs")
return
# Bandwidth test using speedtest-cli
self.log("\nPerforming bandwidth test using speedtest-cli...")
try:
st = speedtest.Speedtest()
st.get_best_server()
download_speed = st.download() # Bits per second
upload_speed = st.upload() # Bits per second
download_mbps = download_speed / 1e6 # Convert to Mbps
upload_mbps = upload_speed / 1e6
self.log(f"Download speed: {download_mbps:.2f} Mbps")
self.log(f"Upload speed: {upload_mbps:.2f} Mbps")
if download_mbps < 1:
self._log_warning("Download speed is low")
if upload_mbps < 0.5:
self._log_warning("Upload speed is low")
except speedtest.ConfigRetrievalError:
self._log_error("Failed to retrieve speedtest configuration")
except Exception as e:
self._log_warning(f"Bandwidth test failed: {e}")
except ImportError:
self._log_error("Required packages are not installed. Please install them using 'pip install requests speedtest-cli'")
except Exception as e:
self._log_error(f"Network connectivity check failed: {e}")
def _step8_environment_variables(self):
self.log("\n===== Environment Variables Check =====")
try:
# Check Python paths
pythonpath = os.environ.get('PYTHONPATH')
if pythonpath:
self.log("\nPYTHONPATH:")
for path in pythonpath.split(os.pathsep):
self.log(f" - {path}")
else:
self.log("\nPYTHONPATH is not set.")
self.log("\nPython sys.path:")
for path in sys.path:
self.log(f" - {path}")
# Check OPENAI_API_KEY
from dotenv import load_dotenv
load_dotenv()
api_key = os.environ.get('OPENAI_API_KEY')
if api_key:
self.log("OPENAI_API_KEY is set after calling load_dotenv()")
if not api_key.startswith('sk-proj-') or len(api_key)<12:
self._log_warning("OPENAI_API_KEY format looks incorrect after calling load_dotenv()")
else:
self._log_warning("OPENAI_API_KEY environment variable is not set after calling load_dotenv()")
except Exception as e:
self._log_error(f"Environment variables check failed: {e}")
def _step9_additional_diagnostics(self):
self.log("\n===== Additional Diagnostics =====")
try:
# Get the site-packages directory paths
import site
site_packages_paths = site.getsitepackages()
if hasattr(site, 'getusersitepackages'):
site_packages_paths.append(site.getusersitepackages())
# Function to check if a path is within site-packages
def is_in_site_packages(path):
return any(os.path.commonpath([path, sp]) == sp for sp in site_packages_paths)
# Check for potential name conflicts in the current directory and sys.path
conflict_names = ['openai.py', 'dotenv.py']
# Check current directory
current_dir = os.getcwd()
for name in conflict_names:
conflict_path = os.path.join(current_dir, name)
if os.path.isfile(conflict_path):
self._log_warning(f"Found '{name}' in the current directory, which may cause import conflicts: {conflict_path}")
# Check sys.path directories
for path in sys.path:
if not path or is_in_site_packages(path):
continue # Skip site-packages and empty paths
for name in conflict_names:
conflict_file = os.path.join(path, name)
if os.path.isfile(conflict_file):
self._log_warning(f"Potential naming conflict: {conflict_file}")
# Check temp directory
try:
with tempfile.NamedTemporaryFile() as tmp:
self.log(f"Temp directory is writable: {os.path.dirname(tmp.name)}")
except Exception as e:
self._log_error(f"Cannot write to temp directory: {e}")
except Exception as e:
self._log_error(f"Additional diagnostics failed: {e}")
if __name__ == "__main__":
diagnostics = Diagnostics()
diagnostics.run()

81
week1/troubleshooting.ipynb

@ -11,7 +11,7 @@
"\n", "\n",
"Setting up a Data Science environment can be challenging because there's a lot going on under the hood. But we will get there.\n", "Setting up a Data Science environment can be challenging because there's a lot going on under the hood. But we will get there.\n",
"\n", "\n",
"And please remember - I'm standing by to help out. Message me or email ed@edwarddonner.com and I'll get on the case.\n" "And please remember - I'm standing by to help out. Message me or email ed@edwarddonner.com and I'll get on the case. The very last cell in this notebook has some diagnostics that will help me figure out what's happening.\n"
] ]
}, },
{ {
@ -34,7 +34,7 @@
"4. Kernel menu >> Restart Kernel and Clear Outputs of All Cells\n", "4. Kernel menu >> Restart Kernel and Clear Outputs of All Cells\n",
"5. Come back to this notebook and try the cell below again.\n", "5. Come back to this notebook and try the cell below again.\n",
"\n", "\n",
"If **that** doesn't work, then please contact me! I'll respond quickly, and we'll figure it out." "If **that** doesn't work, then please contact me! I'll respond quickly, and we'll figure it out. Please run the diagnostics (last cell in this notebook) so I can debug. If you used Anaconda, it might be that for some reason your environment is corrupted, in which case the simplest fix is to use the virtualenv approach instead (Part 2B in the setup guides)."
] ]
}, },
{ {
@ -46,6 +46,9 @@
"source": [ "source": [
"# This should run with no output - no import errors.\n", "# This should run with no output - no import errors.\n",
"# Import errors might indicate that you started jupyter lab without your environment activated? See SETUP part 5.\n", "# Import errors might indicate that you started jupyter lab without your environment activated? See SETUP part 5.\n",
"# Or you might need to restart your Kernel and Jupyter Lab.\n",
"# Or it's possible that something is wrong with Anaconda, in which case we may have to use virtualenv instead.\n",
"# If you're unsure, please run the diagnostics (last cell in this notebook) and then email me at ed@edwarddonner.com\n",
"\n", "\n",
"from openai import OpenAI" "from openai import OpenAI"
] ]
@ -60,7 +63,9 @@
"Let's check your .env file exists and has the OpenAI key set properly inside it. \n", "Let's check your .env file exists and has the OpenAI key set properly inside it. \n",
"Please run this code and check that it prints a successful message, otherwise follow its instructions.\n", "Please run this code and check that it prints a successful message, otherwise follow its instructions.\n",
"\n", "\n",
"Note that the `.env` file won't show up in your Jupyter Lab file browser, because Jupyter hides files that start with a dot for your security; they're considered hidden files. If you need to change the name, you'll need to use a command terminal or File Explorer (PC) / Finder Window (Mac). Ask ChatGPT if that's giving you problems, or email me!" "Note that the `.env` file won't show up in your Jupyter Lab file browser, because Jupyter hides files that start with a dot for your security; they're considered hidden files. If you need to change the name, you'll need to use a command terminal or File Explorer (PC) / Finder Window (Mac). Ask ChatGPT if that's giving you problems, or email me!\n",
"\n",
"If you're having challenges creating the `.env` file, we can also do it with code! See the cell after the next one."
] ]
}, },
{ {
@ -102,6 +107,45 @@
" print(file.name)" " print(file.name)"
] ]
}, },
{
"cell_type": "markdown",
"id": "105f9e0a-9ff4-4344-87c8-e3e41bc50869",
"metadata": {},
"source": [
"## Fallback plan - python code to create the .env file for you\n",
"\n",
"Only run the next cell if you're having problems making the .env file. \n",
"Replace the text in the first line of code with your key from OpenAI."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "ab9ea6ef-49ee-4899-a1c7-75a8bd9ac36b",
"metadata": {},
"outputs": [],
"source": [
"# Only run this code in this cell if you want to have a .env file created for you!\n",
"\n",
"make_me_a_file_with_this_key = \"put your key here inside these quotes.. it should start sk-proj-\"\n",
"\n",
"from pathlib import Path\n",
"\n",
"parent_dir = Path(\"..\")\n",
"env_path = parent_dir / \".env\"\n",
"\n",
"if env_path.exists():\n",
" print(\"There is already a .env file - if you want me to create a new one, please delete the existing one first\")\n",
"else:\n",
" try:\n",
" with env_path.open(mode='w', encoding='utf-8') as env_file:\n",
" env_file.write(f\"OPENAI_API_KEY={make_me_a_file_with_this_key}\")\n",
" print(f\"Successfully created the .env file at {env_path}\")\n",
" print(\"Now rerun the previous cell to confirm that the file is created and the key is correct.\")\n",
" except Exception as e:\n",
" print(f\"An error occurred while creating the .env file: {e}\")"
]
},
{ {
"cell_type": "markdown", "cell_type": "markdown",
"id": "0ba9420d-3bf0-4e08-abac-f2fbf0e9c7f1", "id": "0ba9420d-3bf0-4e08-abac-f2fbf0e9c7f1",
@ -109,7 +153,7 @@
"source": [ "source": [
"# Step 3\n", "# Step 3\n",
"\n", "\n",
"Now let's check that your API key is correct set up in your `.env` file.\n", "Now let's check that your API key is correct set up in your `.env` file, and available using the dotenv package.\n",
"Try running the next cell." "Try running the next cell."
] ]
}, },
@ -181,6 +225,8 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"from openai import OpenAI\n", "from openai import OpenAI\n",
"from dotenv import load_dotenv\n",
"load_dotenv()\n",
"\n", "\n",
"# EITHER:\n", "# EITHER:\n",
"my_api_key = os.getenv(\"OPENAI_API_KEY\")\n", "my_api_key = os.getenv(\"OPENAI_API_KEY\")\n",
@ -222,18 +268,39 @@
"\n", "\n",
"(1) Try pasting your error into ChatGPT or Claude! It's amazing how often they can figure things out\n", "(1) Try pasting your error into ChatGPT or Claude! It's amazing how often they can figure things out\n",
"\n", "\n",
"(2) Contact me! ed@edwarddonner.com\n", "(2) Contact me! Please run the diagnostics in the cell below, then email me your problems to ed@edwarddonner.com\n",
"\n", "\n",
"Thanks so much, and I'm sorry this is giving you bother!" "Thanks so much, and I'm sorry this is giving you bother!"
] ]
}, },
{
"cell_type": "markdown",
"id": "dc83f944-6ce0-4b5c-817f-952676e284ec",
"metadata": {},
"source": [
"# Gathering Essential Diagnostic information\n",
"\n",
"## Please run this next cell to gather some important data\n",
"\n",
"Please run the next cell; it should take a minute or so to run (mostly the network test).\n",
"Rhen email me the output of the last cell to ed@edwarddonner.com. \n",
"Alternatively: this will create a file called report.txt - just attach the file to your email."
]
},
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": null,
"id": "32dea02e-c216-4aed-9f84-934c981d0573", "id": "248204f0-7bad-482a-b715-fb06a3553916",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [] "source": [
"# Run my diagnostics report to collect key information for debugging\n",
"# Please email me the results. Either copy & paste the output, or attach the file report.txt\n",
"\n",
"!pip install -q requests speedtest-cli psutil setuptools\n",
"from diagnostics import Diagnostics\n",
"Diagnostics().run()"
]
} }
], ],
"metadata": { "metadata": {

689
week2/community-contributions/day1-azure-aws-ollama.ipynb

@ -0,0 +1,689 @@
{
"cells": [
{
"cell_type": "markdown",
"id": "06cf3063-9f3e-4551-a0d5-f08d9cabb927",
"metadata": {},
"source": [
"# Welcome to Week 2!\n",
"\n",
"## Frontier Model APIs\n",
"\n",
"In Week 1, we used multiple Frontier LLMs through their Chat UI, and we connected with the OpenAI's API.\n",
"\n",
"Today we'll connect with the APIs for Anthropic and Google, as well as OpenAI."
]
},
{
"cell_type": "markdown",
"id": "2b268b6e-0ba4-461e-af86-74a41f4d681f",
"metadata": {},
"source": [
"<table style=\"margin: 0; text-align: left;\">\n",
" <tr>\n",
" <td style=\"width: 150px; height: 150px; vertical-align: middle;\">\n",
" <img src=\"../important.jpg\" width=\"150\" height=\"150\" style=\"display: block;\" />\n",
" </td>\n",
" <td>\n",
" <h2 style=\"color:#900;\">Important Note - Please read me</h2>\n",
" <span style=\"color:#900;\">I'm continually improving these labs, adding more examples and exercises.\n",
" At the start of each week, it's worth checking you have the latest code.<br/>\n",
" First do a <a href=\"https://chatgpt.com/share/6734e705-3270-8012-a074-421661af6ba9\">git pull and merge your changes as needed</a>. Any problems? Try asking ChatGPT to clarify how to merge - or contact me!<br/><br/>\n",
" After you've pulled the code, from the llm_engineering directory, in an Anaconda prompt (PC) or Terminal (Mac), run:<br/>\n",
" <code>conda env update --f environment.yml --prune</code><br/>\n",
" Or if you used virtualenv rather than Anaconda, then run this from your activated environment in a Powershell (PC) or Terminal (Mac):<br/>\n",
" <code>pip install -r requirements.txt</code>\n",
" <br/>Then restart the kernel (Kernel menu >> Restart Kernel and Clear Outputs Of All Cells) to pick up the changes.\n",
" </span>\n",
" </td>\n",
" </tr>\n",
"</table>\n",
"<table style=\"margin: 0; text-align: left;\">\n",
" <tr>\n",
" <td style=\"width: 150px; height: 150px; vertical-align: middle;\">\n",
" <img src=\"../resources.jpg\" width=\"150\" height=\"150\" style=\"display: block;\" />\n",
" </td>\n",
" <td>\n",
" <h2 style=\"color:#f71;\">Reminder about the resources page</h2>\n",
" <span style=\"color:#f71;\">Here's a link to resources for the course. This includes links to all the slides.<br/>\n",
" <a href=\"https://edwarddonner.com/2024/11/13/llm-engineering-resources/\">https://edwarddonner.com/2024/11/13/llm-engineering-resources/</a><br/>\n",
" Please keep this bookmarked, and I'll continue to add more useful links there over time.\n",
" </span>\n",
" </td>\n",
" </tr>\n",
"</table>"
]
},
{
"cell_type": "markdown",
"id": "85cfe275-4705-4d30-abea-643fbddf1db0",
"metadata": {},
"source": [
"## Setting up your keys\n",
"\n",
"We will use the models through cloud providers, you will need to have credentials for AWS and Azure for this.\n",
"\n",
"When you get your API keys, you need to set them as environment variables by adding them to your `.env` file.\n",
"\n",
"```\n",
"AZURE_OPENAI_API_KEY=xxxx\n",
"AZURE_OPENAI_ENDPOINT=https://example.openai.azure.com\n",
"AWS_ACCESS_KEY_ID=xxxx\n",
"AWS_SECRET_ACCESS_KEY=xxxx\n",
"AWS_SESSION_TOKEN=xxxx\n",
"AWS_REGION=us-west-2\n",
"OPENAI_BASE_URL=https://localhost:11434/v1\n",
"GOOGLE_API_KEY=xxxx\n",
"```\n",
"\n",
"Afterwards, you may need to restart the Jupyter Lab Kernel (the Python process that sits behind this notebook) via the Kernel menu, and then rerun the cells from the top."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "de23bb9e-37c5-4377-9a82-d7b6c648eeb6",
"metadata": {},
"outputs": [],
"source": [
"# imports\n",
"\n",
"import os\n",
"from dotenv import load_dotenv\n",
"from openai import OpenAI, AzureOpenAI\n",
"from dotenv import load_dotenv\n",
"import json\n",
"import boto3\n",
"from IPython.display import Markdown, display, update_display"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "f0a8ab2b-6134-4104-a1bc-c3cd7ea4cd36",
"metadata": {},
"outputs": [],
"source": [
"# import for google\n",
"# in rare cases, this seems to give an error on some systems. Please reach out to me if this happens,\n",
"# or you can feel free to skip Gemini - it's the lowest priority of the frontier models that we use\n",
"\n",
"import google.generativeai"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "c5c0df5e",
"metadata": {},
"outputs": [],
"source": [
"# load the environment variables\n",
"load_dotenv()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1179b4c5-cd1f-4131-a876-4c9f3f38d2ba",
"metadata": {},
"outputs": [],
"source": [
"# Test that AZURE works\n",
"AZURE_MODEL = \"gpt-4o\"\n",
"client_azure = AzureOpenAI(\n",
" api_key=os.getenv('AZURE_OPENAI_API_KEY'),\n",
" azure_endpoint=os.getenv('AZURE_OPENAI_ENDPOINT'),\n",
" api_version=\"2024-08-01-preview\",\n",
")\n",
"messages = [\n",
" {\n",
" \"role\": \"user\",\n",
" \"content\": \"ping\"\n",
" }\n",
"]\n",
"response = client_azure.chat.completions.create(model=AZURE_MODEL, messages=messages)\n",
"print(response.choices[0].message.content)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "0d5fe363",
"metadata": {},
"outputs": [],
"source": [
"# Test that AWS works\n",
"AWS_MODEL = \"anthropic.claude-3-sonnet-20240229-v1:0\"\n",
"session = boto3.Session()\n",
"bedrock = session.client(service_name='bedrock-runtime', region_name='us-east-1')\n",
"# AWS Messages are a bit more complex\n",
"aws_message = {\n",
" \"role\": \"user\",\n",
" \"content\": [\n",
" { \"text\": \"how are you doing\" } \n",
" ],\n",
"}\n",
"response = bedrock.converse(\n",
" modelId=AWS_MODEL,\n",
" inferenceConfig={\n",
" \"maxTokens\": 2000,\n",
" \"temperature\": 0\n",
" },\n",
" messages=[aws_message],\n",
")\n",
"print(response['output']['message']['content'][0]['text'])"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a92f86d4",
"metadata": {},
"outputs": [],
"source": [
"# Test ollama using OpenAI API\n",
"OLLAMA_MODEL='qwen2.5'\n",
"print(os.getenv('OPENAI_BASE_URL'))\n",
"client_ollama = OpenAI(\n",
" base_url=os.getenv('OPENAI_BASE_URL'),\n",
" api_key='123'\n",
" )\n",
"response = client_ollama.chat.completions.create(model=OLLAMA_MODEL, messages=messages)\n",
"print(response.choices[0].message.content)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "797fe7b0-ad43-42d2-acf0-e4f309b112f0",
"metadata": {},
"outputs": [],
"source": [
"# Connect to OpenAI, Anthropic and Google\n",
"# All 3 APIs are similar\n",
"# Having problems with API files? You can use openai = OpenAI(api_key=\"your-key-here\") and same for claude\n",
"# Having problems with Google Gemini setup? Then just skip Gemini; you'll get all the experience you need from GPT and Claude.\n",
"\n",
"google.generativeai.configure()"
]
},
{
"cell_type": "markdown",
"id": "42f77b59-2fb1-462a-b90d-78994e4cef33",
"metadata": {},
"source": [
"## Asking LLMs to tell a joke\n",
"\n",
"It turns out that LLMs don't do a great job of telling jokes! Let's compare a few models.\n",
"Later we will be putting LLMs to better use!\n",
"\n",
"### What information is included in the API\n",
"\n",
"Typically we'll pass to the API:\n",
"- The name of the model that should be used\n",
"- A system message that gives overall context for the role the LLM is playing\n",
"- A user message that provides the actual prompt\n",
"\n",
"There are other parameters that can be used, including **temperature** which is typically between 0 and 1; higher for more random output; lower for more focused and deterministic."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "378a0296-59a2-45c6-82eb-941344d3eeff",
"metadata": {},
"outputs": [],
"source": [
"system_message = \"You are an assistant that is great at telling jokes\"\n",
"user_prompt = \"Tell a light-hearted joke for an audience of Data Scientists\""
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "f4d56a0f-2a3d-484d-9344-0efa6862aff4",
"metadata": {},
"outputs": [],
"source": [
"prompts = [\n",
" {\"role\": \"system\", \"content\": system_message},\n",
" {\"role\": \"user\", \"content\": user_prompt}\n",
" ]"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "3b3879b6-9a55-4fed-a18c-1ea2edfaf397",
"metadata": {},
"outputs": [],
"source": [
"# GPT-4o\n",
"def call_azure(model=AZURE_MODEL, temp=0.5):\n",
" openai = AzureOpenAI(\n",
" api_key=os.getenv('AZURE_OPENAI_API_KEY'),\n",
" azure_endpoint=os.getenv('AZURE_OPENAI_ENDPOINT'),\n",
" api_version=\"2024-08-01-preview\",\n",
" )\n",
" completion = openai.chat.completions.create(model=model, messages=prompts, temperature=temp)\n",
" return completion.choices[0].message.content\n",
"print(call_azure('gpt-4o'))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "3d2d6beb-1b81-466f-8ed1-40bf51e7adbf",
"metadata": {},
"outputs": [],
"source": [
"# GPT-4o-mini\n",
"# Temperature setting controls creativity\n",
"\n",
"print(call_azure('gpt-4o-mini', temp=0.7))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "f1f54beb-823f-4301-98cb-8b9a49f4ce26",
"metadata": {},
"outputs": [],
"source": [
"# GPT-4o\n",
"\n",
"print(call_azure('gpt-4o', temp=0.4))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1ecdb506-9f7c-4539-abae-0e78d7f31b76",
"metadata": {},
"outputs": [],
"source": [
"# AWS with Claude 3.5 Sonnet\n",
"# API needs system message provided separately from user prompt\n",
"# Also adding max_tokens\n",
"\n",
"def call_aws(model=AWS_MODEL, temp=0.5):\n",
" aws_message = {\n",
" \"role\": \"user\",\n",
" \"content\": [\n",
" { \"text\": user_prompt } \n",
" ],\n",
" }\n",
" sys_message = [ { \"text\": system_message } ]\n",
" session = boto3.Session()\n",
" bedrock = session.client(service_name='bedrock-runtime', region_name='us-east-1')\n",
" response = bedrock.converse(\n",
" modelId=model,\n",
" inferenceConfig={\n",
" \"maxTokens\": 2000,\n",
" \"temperature\": temp\n",
" },\n",
" messages=[aws_message],\n",
" system=sys_message\n",
" )\n",
" return response['output']['message']['content'][0]['text']\n",
"print(call_aws(AWS_MODEL))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "769c4017-4b3b-4e64-8da7-ef4dcbe3fd9f",
"metadata": {},
"outputs": [],
"source": [
"# AWS with Claude 3.5 Sonnet\n",
"# Now let's add in streaming back results\n",
"def call_aws_stream(model=AWS_MODEL, temp=0.5):\n",
" aws_message = {\n",
" \"role\": \"user\",\n",
" \"content\": [\n",
" { \"text\": user_prompt } \n",
" ],\n",
" }\n",
" sys_message = [ { \"text\": system_message } ]\n",
" response = bedrock.converse_stream(\n",
" modelId=model,\n",
" inferenceConfig={\n",
" \"maxTokens\": 2000,\n",
" \"temperature\": temp\n",
" },\n",
" system=sys_message,\n",
" messages=[aws_message],\n",
" )\n",
" stream = response.get('stream')\n",
" reply = \"\"\n",
" for event in stream:\n",
" if \"contentBlockDelta\" in event:\n",
" text = event[\"contentBlockDelta\"][\"delta\"]['text']\n",
" print(text, end=\"\", flush=True)\n",
"call_aws_stream(AWS_MODEL, temp=0.7)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "12374cd3",
"metadata": {},
"outputs": [],
"source": [
"# Call Ollama\n",
"def call_ollama_stream(model=OLLAMA_MODEL, temp=0.5):\n",
" openai = OpenAI(\n",
" base_url=os.getenv('OPENAI_BASE_URL'),\n",
" api_key='123'\n",
" )\n",
" stream = openai.chat.completions.create(model=model, messages=prompts, temperature=temp, stream=True)\n",
" for chunk in stream:\n",
" if chunk.choices:\n",
" text = chunk.choices[0].delta.content or ''\n",
" print(text, end=\"\", flush=True)\n",
"call_ollama_stream(OLLAMA_MODEL)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "6df48ce5-70f8-4643-9a50-b0b5bfdb66ad",
"metadata": {},
"outputs": [],
"source": [
"# The API for Gemini has a slightly different structure\n",
"\n",
"gemini = google.generativeai.GenerativeModel(\n",
" model_name='gemini-1.5-flash',\n",
" system_instruction=system_message\n",
")\n",
"response = gemini.generate_content(user_prompt)\n",
"print(response.text)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "83ddb483-4f57-4668-aeea-2aade3a9e573",
"metadata": {},
"outputs": [],
"source": [
"# To be serious! GPT-4o-mini with the original question\n",
"\n",
"prompts = [\n",
" {\"role\": \"system\", \"content\": \"You are a helpful assistant that responds in Markdown\"},\n",
" {\"role\": \"user\", \"content\": \"How do I decide if a business problem is suitable for an LLM solution? Please respond in Markdown.\"}\n",
" ]"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "749f50ab-8ccd-4502-a521-895c3f0808a2",
"metadata": {},
"outputs": [],
"source": [
"# Have it stream back results in markdown\n",
"\n",
"def call_azure_stream(model=AZURE_MODEL, temp=0.5):\n",
" openai = AzureOpenAI(\n",
" api_key=os.getenv('AZURE_OPENAI_API_KEY'),\n",
" azure_endpoint=os.getenv('AZURE_OPENAI_ENDPOINT'),\n",
" api_version=\"2024-08-01-preview\",\n",
" )\n",
" return openai.chat.completions.create(model=model, messages=prompts, temperature=temp, stream=True)\n",
"stream = call_azure_stream('gpt-4o-mini', temp=0.7)\n",
"reply = \"\"\n",
"display_handle = display(Markdown(\"\"), display_id=True)\n",
"for chunk in stream:\n",
" if chunk.choices:\n",
" reply += chunk.choices[0].delta.content or ''\n",
" reply = reply.replace(\"```\",\"\").replace(\"markdown\",\"\")\n",
" update_display(Markdown(reply), display_id=display_handle.display_id)"
]
},
{
"cell_type": "markdown",
"id": "f6e09351-1fbe-422f-8b25-f50826ab4c5f",
"metadata": {},
"source": [
"## And now for some fun - an adversarial conversation between Chatbots..\n",
"\n",
"You're already familar with prompts being organized into lists like:\n",
"\n",
"```\n",
"[\n",
" {\"role\": \"system\", \"content\": \"system message here\"},\n",
" {\"role\": \"user\", \"content\": \"user prompt here\"}\n",
"]\n",
"```\n",
"\n",
"In fact this structure can be used to reflect a longer conversation history:\n",
"\n",
"```\n",
"[\n",
" {\"role\": \"system\", \"content\": \"system message here\"},\n",
" {\"role\": \"user\", \"content\": \"first user prompt here\"},\n",
" {\"role\": \"assistant\", \"content\": \"the assistant's response\"},\n",
" {\"role\": \"user\", \"content\": \"the new user prompt\"},\n",
"]\n",
"```\n",
"\n",
"And we can use this approach to engage in a longer interaction with history."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "bcb54183-45d3-4d08-b5b6-55e380dfdf1b",
"metadata": {},
"outputs": [],
"source": [
"# Let's make a conversation between GPT-4o-mini and Claude-3-haiku\n",
"# We're using cheap versions of models so the costs will be minimal\n",
"\n",
"gpt_model = \"gpt-4o-mini\"\n",
"claude_model = \"anthropic.claude-3-sonnet-20240229-v1:0\"\n",
"\n",
"gpt_system = \"You are a chatbot who is very argumentative; \\\n",
"you disagree with anything in the conversation and you challenge everything, in a snarky way.\"\n",
"\n",
"claude_system = \"You are a very polite, courteous chatbot. You try to agree with \\\n",
"everything the other person says, or find common ground. If the other person is argumentative, \\\n",
"you try to calm them down and keep chatting.\"\n",
"\n",
"gpt_messages = [\"Hi there\"]\n",
"claude_messages = [\"Hi\"]"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1df47dc7-b445-4852-b21b-59f0e6c2030f",
"metadata": {},
"outputs": [],
"source": [
"def call_gpt():\n",
" azure_client = AzureOpenAI(\n",
" api_key=os.getenv('AZURE_OPENAI_API_KEY'),\n",
" azure_endpoint=os.getenv('AZURE_OPENAI_ENDPOINT'),\n",
" api_version=\"2024-08-01-preview\",\n",
" )\n",
" messages = [{\"role\": \"system\", \"content\": gpt_system}]\n",
" for gpt, claude in zip(gpt_messages, claude_messages):\n",
" messages.append({\"role\": \"assistant\", \"content\": gpt})\n",
" messages.append({\"role\": \"user\", \"content\": claude})\n",
" completion = azure_client.chat.completions.create(\n",
" model=gpt_model,\n",
" messages=messages\n",
" )\n",
" return completion.choices[0].message.content"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "9dc6e913-02be-4eb6-9581-ad4b2cffa606",
"metadata": {},
"outputs": [],
"source": [
"call_gpt()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "7d2ed227-48c9-4cad-b146-2c4ecbac9690",
"metadata": {},
"outputs": [],
"source": [
"def call_claude():\n",
" session = boto3.Session()\n",
" bedrock = session.client(service_name='bedrock-runtime', region_name='us-east-1')\n",
" messages = []\n",
" for gpt, claude_message in zip(gpt_messages, claude_messages):\n",
" messages.append({\"role\": \"user\", \"content\": [{\"text\": gpt }]})\n",
" messages.append({\"role\": \"assistant\", \"content\": [{\"text\": claude_message }]})\n",
" messages.append({\"role\": \"user\", \"content\": [{\"text\": gpt_messages[-1] }]})\n",
" response = bedrock.converse(\n",
" modelId=claude_model,\n",
" system=[{\"text\":claude_system}],\n",
" messages=messages,\n",
" inferenceConfig={\n",
" \"maxTokens\": 2000,\n",
" \"temperature\": 0\n",
" },\n",
" )\n",
" return response['output']['message']['content'][0]['text']"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "01395200-8ae9-41f8-9a04-701624d3fd26",
"metadata": {},
"outputs": [],
"source": [
"call_claude()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "08c2279e-62b0-4671-9590-c82eb8d1e1ae",
"metadata": {},
"outputs": [],
"source": [
"call_gpt()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "0275b97f-7f90-4696-bbf5-b6642bd53cbd",
"metadata": {},
"outputs": [],
"source": [
"gpt_messages = [\"Hi there\"]\n",
"claude_messages = [\"Hi\"]\n",
"\n",
"print(f\"GPT:\\n{gpt_messages[0]}\\n\")\n",
"print(f\"Claude:\\n{claude_messages[0]}\\n\")\n",
"\n",
"for i in range(5):\n",
" gpt_next = call_gpt()\n",
" print(f\"GPT:\\n{gpt_next}\\n\")\n",
" gpt_messages.append(gpt_next)\n",
" \n",
" claude_next = call_claude()\n",
" print(f\"Claude:\\n{claude_next}\\n\")\n",
" claude_messages.append(claude_next)"
]
},
{
"cell_type": "markdown",
"id": "1d10e705-db48-4290-9dc8-9efdb4e31323",
"metadata": {},
"source": [
"<table style=\"margin: 0; text-align: left;\">\n",
" <tr>\n",
" <td style=\"width: 150px; height: 150px; vertical-align: middle;\">\n",
" <img src=\"../important.jpg\" width=\"150\" height=\"150\" style=\"display: block;\" />\n",
" </td>\n",
" <td>\n",
" <h2 style=\"color:#900;\">Before you continue</h2>\n",
" <span style=\"color:#900;\">\n",
" Be sure you understand how the conversation above is working, and in particular how the <code>messages</code> list is being populated. Add print statements as needed. Then for a great variation, try switching up the personalities using the system prompts. Perhaps one can be pessimistic, and one optimistic?<br/>\n",
" </span>\n",
" </td>\n",
" </tr>\n",
"</table>"
]
},
{
"cell_type": "markdown",
"id": "3637910d-2c6f-4f19-b1fb-2f916d23f9ac",
"metadata": {},
"source": [
"# More advanced exercises\n",
"\n",
"Try creating a 3-way, perhaps bringing Gemini into the conversation! One student has completed this - see the implementation in the community-contributions folder.\n",
"\n",
"Try doing this yourself before you look at the solutions.\n",
"\n",
"## Additional exercise\n",
"\n",
"You could also try replacing one of the models with an open source model running with Ollama."
]
},
{
"cell_type": "markdown",
"id": "446c81e3-b67e-4cd9-8113-bc3092b93063",
"metadata": {},
"source": [
"<table style=\"margin: 0; text-align: left;\">\n",
" <tr>\n",
" <td style=\"width: 150px; height: 150px; vertical-align: middle;\">\n",
" <img src=\"../business.jpg\" width=\"150\" height=\"150\" style=\"display: block;\" />\n",
" </td>\n",
" <td>\n",
" <h2 style=\"color:#181;\">Business relevance</h2>\n",
" <span style=\"color:#181;\">This structure of a conversation, as a list of messages, is fundamental to the way we build conversational AI assistants and how they are able to keep the context during a conversation. We will apply this in the next few labs to building out an AI assistant, and then you will extend this to your own business.</span>\n",
" </td>\n",
" </tr>\n",
"</table>"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "c23224f6-7008-44ed-a57f-718975f4e291",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": ".venv",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.6"
}
},
"nbformat": 4,
"nbformat_minor": 5
}

92
week2/day1.ipynb

@ -14,6 +14,46 @@
"Today we'll connect with the APIs for Anthropic and Google, as well as OpenAI." "Today we'll connect with the APIs for Anthropic and Google, as well as OpenAI."
] ]
}, },
{
"cell_type": "markdown",
"id": "2b268b6e-0ba4-461e-af86-74a41f4d681f",
"metadata": {},
"source": [
"<table style=\"margin: 0; text-align: left;\">\n",
" <tr>\n",
" <td style=\"width: 150px; height: 150px; vertical-align: middle;\">\n",
" <img src=\"../important.jpg\" width=\"150\" height=\"150\" style=\"display: block;\" />\n",
" </td>\n",
" <td>\n",
" <h2 style=\"color:#900;\">Important Note - Please read me</h2>\n",
" <span style=\"color:#900;\">I'm continually improving these labs, adding more examples and exercises.\n",
" At the start of each week, it's worth checking you have the latest code.<br/>\n",
" First do a <a href=\"https://chatgpt.com/share/6734e705-3270-8012-a074-421661af6ba9\">git pull and merge your changes as needed</a>. Any problems? Try asking ChatGPT to clarify how to merge - or contact me!<br/><br/>\n",
" After you've pulled the code, from the llm_engineering directory, in an Anaconda prompt (PC) or Terminal (Mac), run:<br/>\n",
" <code>conda env update --f environment.yml --prune</code><br/>\n",
" Or if you used virtualenv rather than Anaconda, then run this from your activated environment in a Powershell (PC) or Terminal (Mac):<br/>\n",
" <code>pip install -r requirements.txt</code>\n",
" <br/>Then restart the kernel (Kernel menu >> Restart Kernel and Clear Outputs Of All Cells) to pick up the changes.\n",
" </span>\n",
" </td>\n",
" </tr>\n",
"</table>\n",
"<table style=\"margin: 0; text-align: left;\">\n",
" <tr>\n",
" <td style=\"width: 150px; height: 150px; vertical-align: middle;\">\n",
" <img src=\"../resources.jpg\" width=\"150\" height=\"150\" style=\"display: block;\" />\n",
" </td>\n",
" <td>\n",
" <h2 style=\"color:#f71;\">Reminder about the resources page</h2>\n",
" <span style=\"color:#f71;\">Here's a link to resources for the course. This includes links to all the slides.<br/>\n",
" <a href=\"https://edwarddonner.com/2024/11/13/llm-engineering-resources/\">https://edwarddonner.com/2024/11/13/llm-engineering-resources/</a><br/>\n",
" Please keep this bookmarked, and I'll continue to add more useful links there over time.\n",
" </span>\n",
" </td>\n",
" </tr>\n",
"</table>"
]
},
{ {
"cell_type": "markdown", "cell_type": "markdown",
"id": "85cfe275-4705-4d30-abea-643fbddf1db0", "id": "85cfe275-4705-4d30-abea-643fbddf1db0",
@ -465,30 +505,64 @@
" claude_messages.append(claude_next)" " claude_messages.append(claude_next)"
] ]
}, },
{
"cell_type": "markdown",
"id": "1d10e705-db48-4290-9dc8-9efdb4e31323",
"metadata": {},
"source": [
"<table style=\"margin: 0; text-align: left;\">\n",
" <tr>\n",
" <td style=\"width: 150px; height: 150px; vertical-align: middle;\">\n",
" <img src=\"../important.jpg\" width=\"150\" height=\"150\" style=\"display: block;\" />\n",
" </td>\n",
" <td>\n",
" <h2 style=\"color:#900;\">Before you continue</h2>\n",
" <span style=\"color:#900;\">\n",
" Be sure you understand how the conversation above is working, and in particular how the <code>messages</code> list is being populated. Add print statements as needed. Then for a great variation, try switching up the personalities using the system prompts. Perhaps one can be pessimistic, and one optimistic?<br/>\n",
" </span>\n",
" </td>\n",
" </tr>\n",
"</table>"
]
},
{ {
"cell_type": "markdown", "cell_type": "markdown",
"id": "3637910d-2c6f-4f19-b1fb-2f916d23f9ac", "id": "3637910d-2c6f-4f19-b1fb-2f916d23f9ac",
"metadata": {}, "metadata": {},
"source": [ "source": [
"# See the community-contributions folder\n", "# More advanced exercises\n",
"\n", "\n",
"For a great variation with a 3-way bringing Gemini into the conversation!\n", "Try creating a 3-way, perhaps bringing Gemini into the conversation! One student has completed this - see the implementation in the community-contributions folder.\n",
"\n", "\n",
"Try doing this yourself before you look in the folder.\n", "Try doing this yourself before you look at the solutions.\n",
"\n", "\n",
"## Additional exercise\n", "## Additional exercise\n",
"\n", "\n",
"Try adding in an Ollama model in to the conversation.\n", "You could also try replacing one of the models with an open source model running with Ollama."
"\n", ]
"## Business relevance\n", },
"\n", {
"This structure of a conversation, as a list of messages, is fundamental to the way we build conversational AI assistants and how they are able to keep the context during a conversation. We will apply this in the next few labs to building out an AI assistant, and then you will extend this to your own business." "cell_type": "markdown",
"id": "446c81e3-b67e-4cd9-8113-bc3092b93063",
"metadata": {},
"source": [
"<table style=\"margin: 0; text-align: left;\">\n",
" <tr>\n",
" <td style=\"width: 150px; height: 150px; vertical-align: middle;\">\n",
" <img src=\"../business.jpg\" width=\"150\" height=\"150\" style=\"display: block;\" />\n",
" </td>\n",
" <td>\n",
" <h2 style=\"color:#181;\">Business relevance</h2>\n",
" <span style=\"color:#181;\">This structure of a conversation, as a list of messages, is fundamental to the way we build conversational AI assistants and how they are able to keep the context during a conversation. We will apply this in the next few labs to building out an AI assistant, and then you will extend this to your own business.</span>\n",
" </td>\n",
" </tr>\n",
"</table>"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": null,
"id": "0d86790a-3a6f-4b18-ab0a-bc6107945a27", "id": "c23224f6-7008-44ed-a57f-718975f4e291",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [] "source": []

83
week2/day2.ipynb

@ -172,6 +172,8 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"# The simplicty of gradio. This might appear in \"light mode\" - I'll show you how to make this in dark mode later.\n",
"\n",
"gr.Interface(fn=shout, inputs=\"textbox\", outputs=\"textbox\").launch()" "gr.Interface(fn=shout, inputs=\"textbox\", outputs=\"textbox\").launch()"
] ]
}, },
@ -182,9 +184,55 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"# Adding share=True means that it can be accessed publically\n",
"# A more permanent hosting is available using a platform called Spaces from HuggingFace, which we will touch on next week\n",
"\n",
"gr.Interface(fn=shout, inputs=\"textbox\", outputs=\"textbox\", flagging_mode=\"never\").launch(share=True)" "gr.Interface(fn=shout, inputs=\"textbox\", outputs=\"textbox\", flagging_mode=\"never\").launch(share=True)"
] ]
}, },
{
"cell_type": "code",
"execution_count": null,
"id": "cd87533a-ff3a-4188-8998-5bedd5ba2da3",
"metadata": {},
"outputs": [],
"source": [
"# Adding inbrowser=True opens up a new browser window automatically\n",
"\n",
"gr.Interface(fn=shout, inputs=\"textbox\", outputs=\"textbox\", flagging_mode=\"never\").launch(inbrowser=True)"
]
},
{
"cell_type": "markdown",
"id": "b42ec007-0314-48bf-84a4-a65943649215",
"metadata": {},
"source": [
"## Forcing dark mode\n",
"\n",
"Gradio appears in light mode or dark mode depending on the settings of the browser and computer. There is a way to force gradio to appear in dark mode, but Gradio recommends against this as it should be a user preference (particularly for accessibility reasons). But if you wish to force dark mode for your screens, below is how to do it."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e8129afa-532b-4b15-b93c-aa9cca23a546",
"metadata": {},
"outputs": [],
"source": [
"# Define this variable and then pass js=force_dark_mode when creating the Interface\n",
"\n",
"force_dark_mode = \"\"\"\n",
"function refresh() {\n",
" const url = new URL(window.location);\n",
" if (url.searchParams.get('__theme') !== 'dark') {\n",
" url.searchParams.set('__theme', 'dark');\n",
" window.location.href = url.href;\n",
" }\n",
"}\n",
"\"\"\"\n",
"gr.Interface(fn=shout, inputs=\"textbox\", outputs=\"textbox\", flagging_mode=\"never\", js=force_dark_mode).launch()"
]
},
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": null,
@ -192,6 +240,8 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"# Inputs and Outputs\n",
"\n",
"view = gr.Interface(\n", "view = gr.Interface(\n",
" fn=shout,\n", " fn=shout,\n",
" inputs=[gr.Textbox(label=\"Your message:\", lines=6)],\n", " inputs=[gr.Textbox(label=\"Your message:\", lines=6)],\n",
@ -208,6 +258,8 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"# And now - changing the function from \"shout\" to \"message_gpt\"\n",
"\n",
"view = gr.Interface(\n", "view = gr.Interface(\n",
" fn=message_gpt,\n", " fn=message_gpt,\n",
" inputs=[gr.Textbox(label=\"Your message:\", lines=6)],\n", " inputs=[gr.Textbox(label=\"Your message:\", lines=6)],\n",
@ -224,6 +276,11 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"# Let's use Markdown\n",
"# Are you wondering why it makes any difference to set system_message when it's not referred to in the code below it?\n",
"# I'm taking advantage of system_message being a global variable, used back in the message_gpt function (go take a look)\n",
"# Not a great software engineering practice, but quite sommon during Jupyter Lab R&D!\n",
"\n",
"system_message = \"You are a helpful assistant that responds in markdown\"\n", "system_message = \"You are a helpful assistant that responds in markdown\"\n",
"\n", "\n",
"view = gr.Interface(\n", "view = gr.Interface(\n",
@ -243,6 +300,8 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"# Let's create a call that streams back results\n", "# Let's create a call that streams back results\n",
"# If you'd like a refresher on Generators (the \"yield\" keyword),\n",
"# Please take a look at the Intermediate Python notebook in week1 folder.\n",
"\n", "\n",
"def stream_gpt(prompt):\n", "def stream_gpt(prompt):\n",
" messages = [\n", " messages = [\n",
@ -334,7 +393,9 @@
"\n", "\n",
"There's actually a more elegant way to achieve this (which Python people might call more 'Pythonic'):\n", "There's actually a more elegant way to achieve this (which Python people might call more 'Pythonic'):\n",
"\n", "\n",
"`yield from result`" "`yield from result`\n",
"\n",
"I cover this in more detail in the Intermediate Python notebook in the week1 folder - take a look if you'd like more."
] ]
}, },
{ {
@ -380,6 +441,26 @@
"Now you know how - it's simple!" "Now you know how - it's simple!"
] ]
}, },
{
"cell_type": "markdown",
"id": "92d7c49b-2e0e-45b3-92ce-93ca9f962ef4",
"metadata": {},
"source": [
"<table style=\"margin: 0; text-align: left;\">\n",
" <tr>\n",
" <td style=\"width: 150px; height: 150px; vertical-align: middle;\">\n",
" <img src=\"../important.jpg\" width=\"150\" height=\"150\" style=\"display: block;\" />\n",
" </td>\n",
" <td>\n",
" <h2 style=\"color:#900;\">Before you read the next few cells</h2>\n",
" <span style=\"color:#900;\">\n",
" Try to do this yourself - go back to the company brochure in week1, day5 and add a Gradio UI to the end. Then come and look at the solution.\n",
" </span>\n",
" </td>\n",
" </tr>\n",
"</table>"
]
},
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": null,

18
week2/day3.ipynb

@ -256,11 +256,19 @@
"id": "82a57ee0-b945-48a7-a024-01b56a5d4b3e", "id": "82a57ee0-b945-48a7-a024-01b56a5d4b3e",
"metadata": {}, "metadata": {},
"source": [ "source": [
"# Business Applications\n", "<table style=\"margin: 0; text-align: left;\">\n",
"\n", " <tr>\n",
"Conversational Assistants are of course a hugely common use case for Gen AI, and the latest frontier models are remarkably good at nuanced conversation. And Gradio makes it easy to have a user interface. Another crucial skill we covered is how to use prompting to provide context, information and examples.\n", " <td style=\"width: 150px; height: 150px; vertical-align: middle;\">\n",
"\n", " <img src=\"../business.jpg\" width=\"150\" height=\"150\" style=\"display: block;\" />\n",
"Consider how you could apply an AI Assistant to your business, and make yourself a prototype. Use the system prompt to give context on your business, and set the tone for the LLM." " </td>\n",
" <td>\n",
" <h2 style=\"color:#181;\">Business Applications</h2>\n",
" <span style=\"color:#181;\">Conversational Assistants are of course a hugely common use case for Gen AI, and the latest frontier models are remarkably good at nuanced conversation. And Gradio makes it easy to have a user interface. Another crucial skill we covered is how to use prompting to provide context, information and examples.\n",
"<br/><br/>\n",
"Consider how you could apply an AI Assistant to your business, and make yourself a prototype. Use the system prompt to give context on your business, and set the tone for the LLM.</span>\n",
" </td>\n",
" </tr>\n",
"</table>"
] ]
}, },
{ {

52
week4/day3.ipynb

@ -7,11 +7,53 @@
"source": [ "source": [
"# Code Generator\n", "# Code Generator\n",
"\n", "\n",
"The requirement: use a Frontier model to generate high performance C++ code from Python code\n", "The requirement: use a Frontier model to generate high performance C++ code from Python code\n"
"\n", ]
"# Important Note\n", },
"\n", {
"In the exercise I use GPT-4o and Claude-3.5-Sonnet, which are the slightly higher priced versions. The costs are still low, but if you'd prefer to keep costs ultra low, please make the suggested switches to the models (3 cells down from here)." "cell_type": "markdown",
"id": "d5ccb926-7b49-44a4-99ab-8ef20b5778c0",
"metadata": {},
"source": [
"<table style=\"margin: 0; text-align: left;\">\n",
" <tr>\n",
" <td style=\"width: 150px; height: 150px; vertical-align: middle;\">\n",
" <img src=\"../resources.jpg\" width=\"150\" height=\"150\" style=\"display: block;\" />\n",
" </td>\n",
" <td>\n",
" <h2 style=\"color:#f71;\">Reminder: fetch latest code</h2>\n",
" <span style=\"color:#f71;\">I'm continually improving these labs, adding more examples and exercises.\n",
" At the start of each week, it's worth checking you have the latest code.<br/>\n",
" First do a <a href=\"https://chatgpt.com/share/6734e705-3270-8012-a074-421661af6ba9\">git pull and merge your changes as needed</a>. Any problems? Try asking ChatGPT to clarify how to merge - or contact me!<br/><br/>\n",
" After you've pulled the code, from the llm_engineering directory, in an Anaconda prompt (PC) or Terminal (Mac), run:<br/>\n",
" <code>conda env update --f environment.yml --prune</code><br/>\n",
" Or if you used virtualenv rather than Anaconda, then run this from your activated environment in a Powershell (PC) or Terminal (Mac):<br/>\n",
" <code>pip install -r requirements.txt</code>\n",
" <br/>Then restart the kernel (Kernel menu >> Restart Kernel and Clear Outputs Of All Cells) to pick up the changes.\n",
" </span>\n",
" </td>\n",
" </tr>\n",
"</table>"
]
},
{
"cell_type": "markdown",
"id": "d90e04a2-5b8a-4fd5-9db8-27c02f033313",
"metadata": {},
"source": [
"<table style=\"margin: 0; text-align: left;\">\n",
" <tr>\n",
" <td style=\"width: 150px; height: 150px; vertical-align: middle;\">\n",
" <img src=\"../important.jpg\" width=\"150\" height=\"150\" style=\"display: block;\" />\n",
" </td>\n",
" <td>\n",
" <h1 style=\"color:#900;\">Important Note</h1>\n",
" <span style=\"color:#900;\">\n",
" In this lab, I use GPT-4o and Claude-3.5-Sonnet, which are the slightly higher priced models. The costs are still low, but if you'd prefer to keep costs ultra low, please make the suggested switches to the models (3 cells down from here).\n",
" </span>\n",
" </td>\n",
" </tr>\n",
"</table>"
] ]
}, },
{ {

34
week4/day4.ipynb

@ -11,18 +11,30 @@
"\n", "\n",
"To replicate this, you'll need to set up a HuggingFace endpoint as I do in the video. It's simple to do, and it's quite satisfying to see the results!\n", "To replicate this, you'll need to set up a HuggingFace endpoint as I do in the video. It's simple to do, and it's quite satisfying to see the results!\n",
"\n", "\n",
"It's also an important part of your learning; this is the first example of deploying an open source model to be behind an API. We'll return to this in Week 8, but this should plant a seed in your mind for what's involved in moving open source models into production.\n", "It's also an important part of your learning; this is the first example of deploying an open source model to be behind an API. We'll return to this in Week 8, but this should plant a seed in your mind for what's involved in moving open source models into production."
"\n", ]
"## Important Note\n", },
"\n", {
"If you do decide to use HuggingFace endpoints for this project, you should stop or pause the endpoints when you are done to avoid accruing unnecessary running cost. The costs are very low as long as you only run the endpoint when you're using it. Naviagte to the HuggingFace endpoint UI here:\n", "cell_type": "markdown",
"\n", "id": "22e1567b-33fd-49e7-866e-4b635d15715a",
"https://ui.endpoints.huggingface.co/\n", "metadata": {},
"\n", "source": [
"And open your endpoint, and click Pause to put it on pause so you no longer pay for it. \n", "<table style=\"margin: 0; text-align: left;\">\n",
" <tr>\n",
" <td style=\"width: 150px; height: 150px; vertical-align: middle;\">\n",
" <img src=\"../important.jpg\" width=\"150\" height=\"150\" style=\"display: block;\" />\n",
" </td>\n",
" <td>\n",
" <h1 style=\"color:#900;\">Important - Pause Endpoints when not in use</h1>\n",
" <span style=\"color:#900;\">\n",
" If you do decide to use HuggingFace endpoints for this project, you should stop or pause the endpoints when you are done to avoid accruing unnecessary running cost. The costs are very low as long as you only run the endpoint when you're using it. Navigate to the HuggingFace endpoint UI <a href=\"https://ui.endpoints.huggingface.co/\">here,</a> open your endpoint, and click Pause to put it on pause so you no longer pay for it. \n",
"Many thanks to student John L. for raising this.\n", "Many thanks to student John L. for raising this.\n",
"\n", "<br/><br/>\n",
"In week 8 we will use Modal instead of HuggingFace endpoints; with Modal you only pay for the time that you use it." "In week 8 we will use Modal instead of HuggingFace endpoints; with Modal you only pay for the time that you use it and you should get free credits.\n",
" </span>\n",
" </td>\n",
" </tr>\n",
"</table>"
] ]
}, },
{ {

39
week8/day1.ipynb

@ -9,14 +9,33 @@
"\n", "\n",
"## We have lots to do this week!\n", "## We have lots to do this week!\n",
"\n", "\n",
"We'll move at a faster pace than usual, particularly as you're becoming proficient LLM engineers.\n", "We'll move at a faster pace than usual, particularly as you're becoming proficient LLM engineers.\n"
"\n", ]
"One quick admin thing: I've added a number of packages to the environment.yml file during Sep and Oct. To make sure you have the latest repo with the latest code, it's worth doing this from the `llm_engineering` project folder:\n", },
"\n", {
"```\n", "cell_type": "markdown",
"git pull\n", "id": "b3cf5389-93c5-4523-bc48-78fabb91d8f6",
"conda env update --f environment.yml --prune\n", "metadata": {},
"```" "source": [
"<table style=\"margin: 0; text-align: left;\">\n",
" <tr>\n",
" <td style=\"width: 150px; height: 150px; vertical-align: middle;\">\n",
" <img src=\"../important.jpg\" width=\"150\" height=\"150\" style=\"display: block;\" />\n",
" </td>\n",
" <td>\n",
" <h2 style=\"color:#900;\">Especially important this week: pull the latest</h2>\n",
" <span style=\"color:#900;\">I'm continually improving these labs, adding more examples and exercises.\n",
" At the start of each week, it's worth checking you have the latest code.<br/>\n",
" First do a <a href=\"https://chatgpt.com/share/6734e705-3270-8012-a074-421661af6ba9\">git pull and merge your changes as needed</a>. Any problems? Try asking ChatGPT to clarify how to merge - or contact me!<br/><br/>\n",
" After you've pulled the code, from the llm_engineering directory, in an Anaconda prompt (PC) or Terminal (Mac), run:<br/>\n",
" <code>conda env update --f environment.yml --prune</code><br/>\n",
" Or if you used virtualenv rather than Anaconda, then run this from your activated environment in a Powershell (PC) or Terminal (Mac):<br/>\n",
" <code>pip install -r requirements.txt</code>\n",
" <br/>Then restart the kernel (Kernel menu >> Restart Kernel and Clear Outputs Of All Cells) to pick up the changes.\n",
" </span>\n",
" </td>\n",
" </tr>\n",
"</table>"
] ]
}, },
{ {
@ -43,7 +62,9 @@
"\n", "\n",
"A student on Windows mentioned that on Windows, you might also need to run this command from a command prompt afterwards: \n", "A student on Windows mentioned that on Windows, you might also need to run this command from a command prompt afterwards: \n",
"`modal token new` \n", "`modal token new` \n",
"(Thank you Ed B. for that!)\n" "(Thank you Ed B. for that!)\n",
"\n",
"And I've also heard that in some situations, you might need to restart the Kernel of this jupyter notebook after running this. (Kernel menu >> Restart Kernel and Clear Outputs of All Cells)."
] ]
}, },
{ {

7
week8/day5.ipynb

@ -49,6 +49,9 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"# Updated to change from height to max_height due to change in Gradio v5\n",
"# With much thanks to student Ed B. for raising this\n",
"\n",
"with gr.Blocks(title=\"The Price is Right\", fill_width=True) as ui:\n", "with gr.Blocks(title=\"The Price is Right\", fill_width=True) as ui:\n",
"\n", "\n",
" initial_deal = Deal(product_description=\"Example description\", price=100.0, url=\"https://cnn.com\")\n", " initial_deal = Deal(product_description=\"Example description\", price=100.0, url=\"https://cnn.com\")\n",
@ -69,7 +72,7 @@
" column_widths=[4, 1, 1, 1, 2],\n", " column_widths=[4, 1, 1, 1, 2],\n",
" row_count=10,\n", " row_count=10,\n",
" col_count=5,\n", " col_count=5,\n",
" height=400,\n", " max_height=400,\n",
" )\n", " )\n",
"\n", "\n",
" ui.load(get_table, inputs=[opportunities], outputs=[opportunities_dataframe])\n", " ui.load(get_table, inputs=[opportunities], outputs=[opportunities_dataframe])\n",
@ -111,7 +114,7 @@
" column_widths=[4, 1, 1, 1, 2],\n", " column_widths=[4, 1, 1, 1, 2],\n",
" row_count=10,\n", " row_count=10,\n",
" col_count=5,\n", " col_count=5,\n",
" height=400,\n", " max_height=400,\n",
" )\n", " )\n",
"\n", "\n",
" ui.load(get_table, inputs=[opportunities], outputs=[opportunities_dataframe])\n", " ui.load(get_table, inputs=[opportunities], outputs=[opportunities_dataframe])\n",

2
week8/price_is_right.py

@ -44,7 +44,7 @@ class App:
column_widths=[4, 1, 1, 1, 2], column_widths=[4, 1, 1, 1, 2],
row_count=10, row_count=10,
col_count=5, col_count=5,
height=400, max_height=400,
) )
ui.load(start, inputs=[], outputs=[opportunities_dataframe]) ui.load(start, inputs=[], outputs=[opportunities_dataframe])

Loading…
Cancel
Save