diff --git a/01-python-foundations/my_system_health.py b/01-python-foundations/my_system_health.py new file mode 100644 index 0000000..ceb0cc4 --- /dev/null +++ b/01-python-foundations/my_system_health.py @@ -0,0 +1,32 @@ +import psutil + +try: + threshold = input("Enter the CPU threshold (%): ") + threshold = float(threshold) +except ValueError: + print("That's not a valid number. Please enter something like 75 or 80.5") + exit() + +cpu_usage = psutil.cpu_percent(interval=1) +memory_usage = psutil.virtual_memory().percent +disk_usage = psutil.disk_usage("/").percent + +print("You entered:", threshold) +print("Current CPU usage:", cpu_usage) +print("Current Memory usage:", memory_usage) +print("Current Disk usage:", disk_usage) + +if cpu_usage > threshold: + print("CPU status: WARNING - usage is above threshold") +else: + print("CPU status: Healthy") + +if memory_usage > threshold: + print("Memory status: WARNING - usage is above threshold") +else: + print("Memory status: Healthy") + +if disk_usage > threshold: + print("Disk status: WARNING - usage is above threshold") +else: + print("Disk status: Healthy") \ No newline at end of file diff --git a/02-apis-and-json/call_api.py b/02-apis-and-json/call_api.py index fd86ae6..91e8aba 100644 --- a/02-apis-and-json/call_api.py +++ b/02-apis-and-json/call_api.py @@ -2,7 +2,7 @@ import requests -API_URL = "https://jsonplaceholder.typicode.com/todos/1" +API_URL = "https://jsonplaceholder.typicode.com/todos/2" def fetch_todo(url): @@ -17,8 +17,8 @@ def main(): for key, value in todo.items(): print(f"{key:10}: {value}") - if todo.get("userId") == 1: - print("\n>> This todo belongs to user 1") + if todo.get("userId") == 2: + print("\n>> This todo belongs to user 2") if __name__ == "__main__": diff --git a/02-apis-and-json/github_user.py b/02-apis-and-json/github_user.py new file mode 100644 index 0000000..e95948c --- /dev/null +++ b/02-apis-and-json/github_user.py @@ -0,0 +1,24 @@ +import json +import requests + +username = input("Enter a GitHub username: ") +url = f"https://api.github.com/users/{username}" + +try: + response = requests.get(url, timeout=10) + response.raise_for_status() +except requests.exceptions.HTTPError: + print(f"Could not find a GitHub user called '{username}'.") + exit() + +data = response.json() + +print("Name :", data.get("name")) +print("Public repos:", data.get("public_repos")) +print("Followers :", data.get("followers")) +print("Location :", data.get("location")) + +with open("github_user.json", "w", encoding="utf-8") as f: + json.dump(data, f, indent=2) + +print("\nSaved full response to github_user.json") \ No newline at end of file diff --git a/02-apis-and-json/stock_market_api.py b/02-apis-and-json/stock_market_api.py index 27d8643..5652ba9 100644 --- a/02-apis-and-json/stock_market_api.py +++ b/02-apis-and-json/stock_market_api.py @@ -24,7 +24,7 @@ def get_daily_series(symbol, api_key): def main(): api_key = os.environ.get("ALPHAVANTAGE_API_KEY") if not api_key: - print("Set ALPHAVANTAGE_API_KEY first: export ALPHAVANTAGE_API_KEY=...") + print("Set ALPHAVANTAGE_API_KEY first: export ALPHAVANTAGE_API_KEY=MCHYRHM0ME21PCKH") sys.exit(1) symbol = input("Enter a stock symbol (e.g. IBM, AMZN, GOOGL): ").strip().upper() diff --git a/03-file-handling-and-logs/my_log_analyzer.py b/03-file-handling-and-logs/my_log_analyzer.py new file mode 100644 index 0000000..2c89574 --- /dev/null +++ b/03-file-handling-and-logs/my_log_analyzer.py @@ -0,0 +1,40 @@ +import json + +LOG_FILE = "app.log" + +try: + with open(LOG_FILE, "r", encoding="utf-8") as f: + lines = f.readlines() +except FileNotFoundError: + print(f"Log file not found: {LOG_FILE}") + exit() + +print("Total lines:", len(lines)) + +info_count = 0 +warning_count = 0 +error_count = 0 + +for line in lines: + words = line.split() + if "INFO" in words: + info_count += 1 + if "WARNING" in words: + warning_count += 1 + if "ERROR" in words: + error_count += 1 + +print("INFO :", info_count) +print("WARNING:", warning_count) +print("ERROR :", error_count) + +summary = { + "INFO": info_count, + "WARNING": warning_count, + "ERROR": error_count, +} + +with open("log_summary.json", "w", encoding="utf-8") as f: + json.dump(summary, f, indent=2) + +print("\nSaved summary to log_summary.json") \ No newline at end of file diff --git a/04-object-oriented-python/log_analyzer_oop.py b/04-object-oriented-python/log_analyzer_oop.py index e47f793..6e15443 100644 --- a/04-object-oriented-python/log_analyzer_oop.py +++ b/04-object-oriented-python/log_analyzer_oop.py @@ -9,7 +9,12 @@ class LogAnalyzer: def __init__(self, log_file): self.log_file = log_file - self.counts = {level: 0 for level in LEVELS} + self.counts = { + "INFO": 0, + "WARNING": 0, + "ERROR": 0, + "UNKNOWN": 0, + } def read_logs(self): try: @@ -32,7 +37,9 @@ def analyze(self, lines): self.counts["UNKNOWN"] += 1 return self.counts - def write_summary(self, path="log_counts.json"): + def write_summary(self, path=None): + if path is None: + path = Path(self.log_file).with_suffix(".json") with open(path, "w", encoding="utf-8") as f: json.dump(self.counts, f, indent=2) @@ -47,6 +54,8 @@ def main(): return result = analyzer.analyze(lines) + analyzer.write_summary("custom_log_counts.json") + print("Log Analysis Summary:") for level, count in result.items(): print(f" {level:7}: {count}") diff --git a/05-cli-tools-argparse/my_log_analyzer_cli.py b/05-cli-tools-argparse/my_log_analyzer_cli.py new file mode 100644 index 0000000..268a23b --- /dev/null +++ b/05-cli-tools-argparse/my_log_analyzer_cli.py @@ -0,0 +1,52 @@ +import argparse +import json +import sys +from pathlib import Path + +LEVELS = ("INFO", "WARNING", "ERROR") + +parser = argparse.ArgumentParser(description="Analyze a log file for INFO/WARNING/ERROR counts.") +parser.add_argument("--file", required=True, help="path to the log file") +parser.add_argument("--out", help="write the summary to this JSON file") +parser.add_argument("--level", choices=LEVELS, help="show the count for only this level") + +args = parser.parse_args() + +log_path = Path(args.file) +if not log_path.is_file(): + print(f"Error: log file not found: {args.file}", file=sys.stderr) + sys.exit(2) + +with open(log_path, "r", encoding="utf-8") as f: + lines = f.readlines() + +info_count = 0 +warning_count = 0 +error_count = 0 + +for line in lines: + words = line.split() + if "INFO" in words: + info_count += 1 + if "WARNING" in words: + warning_count += 1 + if "ERROR" in words: + error_count += 1 + +counts = { + "INFO": info_count, + "WARNING": warning_count, + "ERROR": error_count, +} + +if args.level: + print(f"{args.level}: {counts[args.level]}") +else: + print("INFO :", info_count) + print("WARNING:", warning_count) + print("ERROR :", error_count) + +if args.out: + with open(args.out, "w", encoding="utf-8") as f: + json.dump(counts, f, indent=2) + print(f"Wrote summary to {args.out}") \ No newline at end of file diff --git a/05-cli-tools-argparse/my_summary.json b/05-cli-tools-argparse/my_summary.json new file mode 100644 index 0000000..7481550 --- /dev/null +++ b/05-cli-tools-argparse/my_summary.json @@ -0,0 +1,5 @@ +{ + "INFO": 10, + "WARNING": 2, + "ERROR": 3 +} \ No newline at end of file diff --git a/05-cli-tools-argparse/summary.json b/05-cli-tools-argparse/summary.json new file mode 100644 index 0000000..7481550 --- /dev/null +++ b/05-cli-tools-argparse/summary.json @@ -0,0 +1,5 @@ +{ + "INFO": 10, + "WARNING": 2, + "ERROR": 3 +} \ No newline at end of file diff --git a/06-aws-automation-boto3/cdk_demo/cdk.json b/06-aws-automation-boto3/cdk_demo/cdk.json new file mode 100644 index 0000000..03d0169 --- /dev/null +++ b/06-aws-automation-boto3/cdk_demo/cdk.json @@ -0,0 +1,3 @@ +{ + "app": "python app.py" +} \ No newline at end of file diff --git a/README.md b/README.md index 0585698..948c7fc 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,9 @@ # Python for DevOps + Agentic AI +

+ Elvis's Contribution Graph +

+ Learn to use Python for real DevOps work — automation, cloud operations, log analysis, internal tooling, and local AI agents. diff --git a/capstone/README.md b/capstone/README.md index cdeb73d..f1e5563 100644 --- a/capstone/README.md +++ b/capstone/README.md @@ -6,6 +6,61 @@ interview-ready. --- +## My implementation + +A FastAPI service — "DevOps Intelligence API" — covering the full "Impressive" +tier: log analysis, system metrics, AWS resource visibility, and an +AI-assisted log investigation endpoint backed by a local Ollama model. + +### Endpoints + +| Method | Path | What it does | +| ------ | -------------- | -------------------------------------------------- | +| GET | `/health` | Liveness check | +| GET | `/logs` | INFO/WARNING/ERROR counts for a log file | +| GET | `/metrics` | Current CPU, memory, and disk usage | +| GET | `/aws/s3` | S3 buckets grouped by age | +| GET | `/aws/ec2` | EC2 instances and their state | +| GET | `/aws/report` | Combined S3 + EC2 inventory | +| POST | `/ai/analyze` | Plain-English log summary from a local LLM agent | + +### Setup + +```bash +cd capstone +python -m venv .venv +.venv\Scripts\activate # Windows +# source .venv/bin/activate # macOS/Linux + +pip install -r requirements.txt +``` + +### Run + +```bash +uvicorn app.main:app --reload +``` + +The API is then available at `http://127.0.0.1:8000` (interactive docs at +`/docs`). `/logs` and `/ai/analyze` default to `sample_logs/app.log` if no +`file` is given. + +The `/aws/*` endpoints need AWS credentials available to `boto3` (e.g. via +`aws configure` or environment variables). `/ai/analyze` needs a local +[Ollama](https://ollama.com) instance running with the model set in +`OLLAMA_MODEL` (defaults to `qwen3.6:latest`) pulled and available. + +### Tests + +```bash +pytest +``` + +All 16 tests run without live AWS credentials or a running Ollama instance — +AWS and AI calls are mocked. + +--- + ## The story these modules tell Every module in this course builds one capability. The capstone assembles them: diff --git a/capstone/STAR.md b/capstone/STAR.md index 16e5e98..961aac3 100644 --- a/capstone/STAR.md +++ b/capstone/STAR.md @@ -1,35 +1,46 @@ # My Capstone — S.T.A.R Explanation -Fill this in with your own words. A lot of people struggle in interviews not for -lack of skills, but because they can't explain their work clearly. Bullet points -are fine. - ## Situation - -- +- Our team's application logs and infrastructure health were only checked + manually — someone had to SSH in, tail log files, and separately check AWS + and system metrics whenever something seemed off. That was slow, easy to + forget, and gave no consistent way to check status on demand. ## Task - -- +- I was responsible for automating this with Python: parse and summarize + application logs, expose system health metrics, surface AWS resource + inventory, and make all of it available through a single, reusable service + instead of one-off scripts. ## Action - -- +- Wrote a log parser that reads a log file and counts INFO/WARNING/ERROR + occurrences using whole-word matching, then wrapped it in an + `analyze_log_file` function returning structured counts. +- Built a `psutil`-based metrics service reporting CPU, memory, and disk + usage, with a configurable CPU threshold that flags the system as healthy + or under high load. +- Used `boto3` to build an AWS inventory service: S3 buckets grouped by age + (new vs. old, based on a configurable day threshold) and EC2 instances with + their current state. +- Exposed all of it through a FastAPI service (`/health`, `/logs`, `/metrics`, + `/aws/s3`, `/aws/ec2`, `/aws/report`) with routers, services, and Pydantic + schemas kept in separate layers, and consistent HTTP error mapping + (404/400/500/502) for missing files, bad input, and AWS/API failures. +- Added a local AI layer: a LangChain/LangGraph agent running against a local + Ollama model, given a tool that returns the exact deterministic log counts + so it can't invent statistics, exposed via `POST /ai/analyze`. Added + path-traversal protection so the endpoint can only read `.log` files inside + the approved log directory. +- Wrote 16 unit tests covering the log parser, metrics service, AWS service + (mocking `boto3` clients so no real AWS calls are made in tests), and the + AI router's path-resolution and security checks. ## Result - -- - ---- - -### Example (for reference — replace with your own) - -- Situation: Application logs grew daily and manually scanning them for errors - was slow and error-prone. -- Task: Automate log analysis with Python so issues surface quickly. -- Action: Wrote a log parser that counts INFO/WARNING/ERROR, refactored it into - a class, added an `argparse` CLI, then exposed it via a FastAPI `/logs` - endpoint alongside `/metrics` and `/health`. Added a local LangGraph + Ollama - agent to summarize logs in plain English. -- Result: Reduced manual log review effort and gave the team quick, on-demand - visibility into application health through a single API. +- Replaced manual log/metrics/AWS checks with a single running API that + answers each question in one request instead of several manual steps. +- The AI endpoint gives a plain-English read of what a log file shows, + grounded in the same deterministic counts the API reports elsewhere, which + makes it trustworthy enough to actually rely on rather than just a demo. +- All 16 tests pass and the AWS/AI tests run without needing live AWS + credentials or a running Ollama instance, so the suite is safe to run in + CI. diff --git a/capstone/app/__init__.py b/capstone/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/capstone/app/main.py b/capstone/app/main.py new file mode 100644 index 0000000..5019b30 --- /dev/null +++ b/capstone/app/main.py @@ -0,0 +1,33 @@ +from fastapi import FastAPI + +from app.routers import ai, aws, logs, metrics + + +app = FastAPI( + title="DevOps Intelligence API", + description=( + "A Python DevOps service for log analysis, system metrics, " + "AWS resource visibility, and AI-assisted investigation." + ), + version="1.0.0", +) + + +@app.get("/") +def root(): + return { + "name": "DevOps Intelligence API", + "version": app.version, + "status": "running", + } + + +@app.get("/health") +def health(): + return {"status": "ok"} + + +app.include_router(logs.router) +app.include_router(metrics.router) +app.include_router(aws.router) +app.include_router(ai.router) \ No newline at end of file diff --git a/capstone/app/routers/__init__.py b/capstone/app/routers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/capstone/app/routers/ai.py b/capstone/app/routers/ai.py new file mode 100644 index 0000000..2a32611 --- /dev/null +++ b/capstone/app/routers/ai.py @@ -0,0 +1,63 @@ +from pathlib import Path + +from fastapi import APIRouter, HTTPException + +from app.schemas.ai import AIAnalyzeRequest, AIAnalyzeResponse +from app.services.ai_service import analyze_logs_with_ai + + +router = APIRouter(prefix="/ai", tags=["ai"]) + +BASE_DIR = Path(__file__).resolve().parents[2] +LOG_DIR = BASE_DIR / "sample_logs" +DEFAULT_LOG = LOG_DIR / "app.log" + + +def resolve_log_path(file: str | None) -> Path: + """Resolve a log filename without allowing access outside LOG_DIR.""" + if not file: + return DEFAULT_LOG + + requested_path = Path(file) + + if requested_path.is_absolute(): + raise ValueError("Only log filenames inside the log directory are allowed.") + + candidate = (LOG_DIR / requested_path).resolve() + + if candidate != LOG_DIR and LOG_DIR not in candidate.parents: + raise ValueError("Log file must be inside the log directory.") + + if candidate.suffix.lower() != ".log": + raise ValueError("Only .log files are allowed.") + + return candidate + + +@router.post( + "/analyze", + response_model=AIAnalyzeResponse, +) +def analyze_with_ai(request: AIAnalyzeRequest): + """Analyze an approved log file using the local AI agent.""" + try: + path = resolve_log_path(request.file) + return analyze_logs_with_ai(path) + + except FileNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + except ConnectionError as exc: + raise HTTPException( + status_code=503, + detail=f"AI service unavailable: {exc}", + ) from exc + + except Exception as exc: + raise HTTPException( + status_code=503, + detail=f"AI analysis failed: {exc}", + ) from exc \ No newline at end of file diff --git a/capstone/app/routers/aws.py b/capstone/app/routers/aws.py new file mode 100644 index 0000000..6c434ab --- /dev/null +++ b/capstone/app/routers/aws.py @@ -0,0 +1,49 @@ +from botocore.exceptions import BotoCoreError, ClientError +from fastapi import APIRouter, HTTPException + +from app.services.aws_service import ( + get_aws_report, + get_ec2_report, + get_s3_report, +) + + +router = APIRouter(prefix="/aws", tags=["aws"]) + + +@router.get("/s3") +def get_s3(): + """Return the S3 bucket inventory.""" + try: + return get_s3_report() + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + except (BotoCoreError, ClientError) as exc: + raise HTTPException( + status_code=502, + detail=f"AWS S3 error: {exc}", + ) from exc + + +@router.get("/ec2") +def get_ec2(): + """Return the EC2 instance inventory.""" + try: + return get_ec2_report() + except (BotoCoreError, ClientError) as exc: + raise HTTPException( + status_code=502, + detail=f"AWS EC2 error: {exc}", + ) from exc + + +@router.get("/report") +def get_report(): + """Return the combined AWS resource inventory.""" + try: + return get_aws_report() + except (BotoCoreError, ClientError) as exc: + raise HTTPException( + status_code=502, + detail=f"AWS error: {exc}", + ) from exc \ No newline at end of file diff --git a/capstone/app/routers/logs.py b/capstone/app/routers/logs.py new file mode 100644 index 0000000..94eca7b --- /dev/null +++ b/capstone/app/routers/logs.py @@ -0,0 +1,28 @@ +from pathlib import Path + +from fastapi import APIRouter, HTTPException + +from app.services.log_service import analyze_log_file + + +router = APIRouter(tags=["logs"]) + +DEFAULT_LOG = Path(__file__).resolve().parents[2] / "sample_logs" / "app.log" + + +@router.get("/logs") +def get_log_summary(file: str | None = None): + """Analyze a log file and return deterministic log statistics.""" + path = Path(file) if file else DEFAULT_LOG + + try: + return analyze_log_file(path) + except FileNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + except OSError as exc: + raise HTTPException( + status_code=500, + detail=f"Unable to read log file: {exc}", + ) from exc \ No newline at end of file diff --git a/capstone/app/routers/metrics.py b/capstone/app/routers/metrics.py new file mode 100644 index 0000000..1c9c8df --- /dev/null +++ b/capstone/app/routers/metrics.py @@ -0,0 +1,20 @@ +from fastapi import APIRouter, HTTPException + +from app.services.metrics_service import get_system_metrics + + +router = APIRouter(tags=["metrics"]) + + +@router.get("/metrics") +def get_metrics(cpu_threshold: float = 85.0): + """Return current system metrics.""" + try: + return get_system_metrics(cpu_threshold) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + except Exception as exc: + raise HTTPException( + status_code=500, + detail=f"Could not read system metrics: {exc}", + ) from exc \ No newline at end of file diff --git a/capstone/app/schemas/__init__.py b/capstone/app/schemas/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/capstone/app/schemas/ai.py b/capstone/app/schemas/ai.py new file mode 100644 index 0000000..de755ef --- /dev/null +++ b/capstone/app/schemas/ai.py @@ -0,0 +1,12 @@ +from pydantic import BaseModel + + +class AIAnalyzeRequest(BaseModel): + file: str | None = None + + +class AIAnalyzeResponse(BaseModel): + log_file: str + counts: dict[str, int] + analysis: str + model: str \ No newline at end of file diff --git a/capstone/app/services/__init__.py b/capstone/app/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/capstone/app/services/ai_service.py b/capstone/app/services/ai_service.py new file mode 100644 index 0000000..a0065ee --- /dev/null +++ b/capstone/app/services/ai_service.py @@ -0,0 +1,129 @@ +import os +from pathlib import Path + +from langchain.agents import create_agent +from langchain_core.tools import tool +from langchain_ollama import ChatOllama + +from app.services.log_service import count_log_levels, read_log_file + + +DEFAULT_MODEL = "qwen3.6:latest" +OLLAMA_BASE_URL = "http://localhost:11434" + + +SYSTEM_PROMPT = ( + "You are a DevOps log analysis assistant. " + "Use only the information provided in the log content and the deterministic " + "statistics generated by the Python application. " + "Never invent events, causes, infrastructure details, or statistics. " + "Always use the analyze_log_file tool to obtain the exact INFO, WARNING, " + "and ERROR counts. " + "State the exact counts first. " + "Then explain what the actual log entries indicate. " + "When making an inference, clearly label it as an inference. " + "Do not introduce hypothetical examples that are not present in the log. " + "Suggest investigation ideas only when they are directly supported by the " + "actual log entries. " + "Never claim that an action was performed." +) + + +@tool +def analyze_log_file(path: str) -> str: + """Read a log file and return exact INFO, WARNING and ERROR counts.""" + text = read_log_file(path) + counts = count_log_levels(text) + + return ", ".join( + f"{level}={counts[level]}" + for level in ("INFO", "WARNING", "ERROR") + ) + + +def get_model_name() -> str: + """Return the configured Ollama model name.""" + return os.environ.get("OLLAMA_MODEL", DEFAULT_MODEL) + + +def make_agent(): + """Create the local DevOps log-analysis agent.""" + model = ChatOllama( + model=get_model_name(), + base_url=OLLAMA_BASE_URL, + temperature=0, + ) + + return create_agent( + model, + tools=[analyze_log_file], + system_prompt=SYSTEM_PROMPT, + ) + + +_agent = None + + +def get_agent(): + """Create the agent once and reuse it for subsequent requests.""" + global _agent + + if _agent is None: + _agent = make_agent() + + return _agent + + +def analyze_logs_with_ai(path: str | Path) -> dict: + """ + Analyze a log file using deterministic Python statistics and a local LLM. + """ + log_path = Path(path) + + if not log_path.exists(): + raise FileNotFoundError(f"Log file not found: {log_path}") + + if not log_path.is_file(): + raise ValueError(f"Log path is not a file: {log_path}") + + text = read_log_file(log_path) + counts = count_log_levels(text) + + agent = get_agent() + + prompt = ( + f"Analyze this DevOps log file: {log_path}\n\n" + "Use the analyze_log_file tool first to verify the exact counts.\n\n" + "Here is the complete log content. Use only this content when " + "explaining what happened:\n\n" + f"{text}" + ) + + result = agent.invoke( + { + "messages": [ + { + "role": "user", + "content": prompt, + } + ] + }, + {"recursion_limit": 10}, + ) + + messages = result.get("messages", []) + + if not messages: + raise RuntimeError("AI agent returned no messages.") + + analysis = messages[-1].content + + if not isinstance(analysis, str): + analysis = str(analysis) + + return { + "log_file": str(log_path), + "counts": counts, + "analysis": analysis.strip(), + "model": get_model_name(), + } \ No newline at end of file diff --git a/capstone/app/services/aws_service.py b/capstone/app/services/aws_service.py new file mode 100644 index 0000000..65a87c6 --- /dev/null +++ b/capstone/app/services/aws_service.py @@ -0,0 +1,75 @@ +from datetime import datetime, timedelta, timezone + +import boto3 +from botocore.exceptions import BotoCoreError, ClientError + + +def _get_s3_client(): + """Create a Boto3 S3 client using the active AWS credentials.""" + return boto3.client("s3") + + +def _get_ec2_client(): + """Create a Boto3 EC2 client for the configured AWS region.""" + return boto3.client("ec2") + + +def get_s3_report(days_threshold: int = 90) -> dict: + """ + Return an inventory of S3 buckets grouped by age. + + Buckets older than the threshold are considered old. + """ + if days_threshold < 0: + raise ValueError("days_threshold cannot be negative") + + response = _get_s3_client().list_buckets() + buckets = response.get("Buckets", []) + + now = datetime.now(timezone.utc) + cutoff = now - timedelta(days=days_threshold) + + new_buckets = [] + old_buckets = [] + + for bucket in buckets: + name = bucket["Name"] + creation_date = bucket["CreationDate"] + + if creation_date < cutoff: + old_buckets.append(name) + else: + new_buckets.append(name) + + return { + "total_buckets": len(buckets), + "new_buckets": new_buckets, + "old_buckets": old_buckets, + "age_threshold_days": days_threshold, + } + + +def get_ec2_report() -> list[dict]: + """Return EC2 instance IDs and their current states.""" + response = _get_ec2_client().describe_instances() + + instances = [] + + for reservation in response.get("Reservations", []): + for instance in reservation.get("Instances", []): + instances.append( + { + "instance_id": instance["InstanceId"], + "state": instance.get("State", {}).get("Name", "unknown"), + } + ) + + return instances + + +def get_aws_report() -> dict: + """Return a combined S3 and EC2 inventory report.""" + return { + "s3": get_s3_report(), + "ec2": get_ec2_report(), + } \ No newline at end of file diff --git a/capstone/app/services/log_service.py b/capstone/app/services/log_service.py new file mode 100644 index 0000000..ad1290c --- /dev/null +++ b/capstone/app/services/log_service.py @@ -0,0 +1,44 @@ +from collections import Counter +from pathlib import Path + + +LEVELS = ("INFO", "WARNING", "ERROR") + + +def read_log_file(path: str | Path) -> str: + """Read a UTF-8 log file and return its contents.""" + log_path = Path(path) + + if not log_path.exists(): + raise FileNotFoundError(f"Log file not found: {log_path}") + + if not log_path.is_file(): + raise ValueError(f"Log path is not a file: {log_path}") + + return log_path.read_text(encoding="utf-8") + + +def count_log_levels(text: str) -> dict[str, int]: + """Count INFO, WARNING and ERROR occurrences by whole-word matching.""" + counts = Counter() + + for line in text.splitlines(): + tokens = set(line.split()) + + for level in LEVELS: + if level in tokens: + counts[level] += 1 + + return {level: counts.get(level, 0) for level in LEVELS} + + +def analyze_log_file(path: str | Path) -> dict: + """Read and analyze a log file.""" + text = read_log_file(path) + counts = count_log_levels(text) + + return { + "log_file": str(path), + "counts": counts, + "total_lines": len(text.splitlines()), + } \ No newline at end of file diff --git a/capstone/app/services/metrics_service.py b/capstone/app/services/metrics_service.py new file mode 100644 index 0000000..65bef0c --- /dev/null +++ b/capstone/app/services/metrics_service.py @@ -0,0 +1,35 @@ +import psutil + + +DEFAULT_CPU_THRESHOLD = 85.0 + + +def get_system_metrics( + cpu_threshold: float = DEFAULT_CPU_THRESHOLD, +) -> dict: + """ + Return current CPU, memory, and disk usage. + + The CPU threshold determines whether the system is reported as healthy + or experiencing high CPU usage. + """ + if not 0 <= cpu_threshold <= 100: + raise ValueError("cpu_threshold must be between 0 and 100") + + cpu_percent = psutil.cpu_percent(interval=0.5) + memory_percent = psutil.virtual_memory().percent + disk_percent = psutil.disk_usage("/").percent + + status = ( + "High CPU" + if cpu_percent > cpu_threshold + else "Healthy" + ) + + return { + "cpu_percentage": cpu_percent, + "memory_percentage": memory_percent, + "disk_percentage": disk_percent, + "cpu_threshold": cpu_threshold, + "system_status": status, + } \ No newline at end of file diff --git a/capstone/requirements.txt b/capstone/requirements.txt new file mode 100644 index 0000000..879eb0f --- /dev/null +++ b/capstone/requirements.txt @@ -0,0 +1,9 @@ +fastapi==0.139.2 +uvicorn[standard] +psutil +boto3 +botocore[crt] +langchain +langchain-core +langchain-ollama +langgraph \ No newline at end of file diff --git a/capstone/sample_logs/app.log b/capstone/sample_logs/app.log new file mode 100644 index 0000000..056314a --- /dev/null +++ b/capstone/sample_logs/app.log @@ -0,0 +1,21 @@ +2025-01-10 09:00:01 INFO Application started successfully +2025-01-10 09:00:05 INFO Connecting to database +2025-01-10 09:00:07 INFO Database connection established + +2025-01-10 09:05:12 WARNING High memory usage detected +2025-01-10 09:05:15 INFO Memory usage back to normal + +2025-01-10 09:10:22 ERROR Failed to fetch user data +2025-01-10 09:10:25 ERROR Database timeout occurred + +2025-01-10 09:15:30 INFO Retrying database connection +2025-01-10 09:15:32 INFO Database connection successful + +2025-01-10 09:20:45 WARNING Disk usage above 75% +2025-01-10 09:20:50 INFO Disk cleanup initiated + +2025-01-10 09:25:10 ERROR Unable to write logs to disk +2025-01-10 09:25:15 INFO Log rotation completed + +2025-01-10 09:30:00 INFO Application shutdown initiated +2025-01-10 09:30:05 INFO Application stopped diff --git a/capstone/tests/test_ai.py b/capstone/tests/test_ai.py new file mode 100644 index 0000000..8b9331a --- /dev/null +++ b/capstone/tests/test_ai.py @@ -0,0 +1,57 @@ +from unittest.mock import patch + +import pytest + +from app.routers.ai import resolve_log_path + + +def test_resolve_default_log(): + path = resolve_log_path(None) + + assert path.name == "app.log" + assert path.suffix == ".log" + + +def test_resolve_valid_log(): + path = resolve_log_path("app.log") + + assert path.name == "app.log" + + +def test_reject_absolute_path(): + with pytest.raises(ValueError): + resolve_log_path("C:\\Windows\\System32\\secret.log") + + +def test_reject_path_traversal(): + with pytest.raises(ValueError): + resolve_log_path("../../secret.log") + + +def test_reject_non_log_file(): + with pytest.raises(ValueError): + resolve_log_path("notes.txt") + + +@patch("app.routers.ai.analyze_logs_with_ai") +def test_ai_router_uses_resolved_path(mock_analyze): + mock_analyze.return_value = { + "log_file": "sample_logs\\app.log", + "counts": { + "INFO": 10, + "WARNING": 2, + "ERROR": 3, + }, + "analysis": "Test analysis", + "model": "qwen3.6:latest", + } + + from app.routers.ai import analyze_with_ai + from app.schemas.ai import AIAnalyzeRequest + + response = analyze_with_ai( + AIAnalyzeRequest(file="app.log") + ) + + assert response["counts"]["ERROR"] == 3 + mock_analyze.assert_called_once() \ No newline at end of file diff --git a/capstone/tests/test_aws.py b/capstone/tests/test_aws.py new file mode 100644 index 0000000..7c8cfad --- /dev/null +++ b/capstone/tests/test_aws.py @@ -0,0 +1,63 @@ +from unittest.mock import MagicMock, patch + +import pytest + +from app.services.aws_service import get_ec2_report, get_s3_report + + +@patch("app.services.aws_service._get_s3_client") +def test_get_s3_report(mock_get_client): + mock_client = MagicMock() + + mock_client.list_buckets.return_value = { + "Buckets": [ + { + "Name": "new-bucket", + "CreationDate": __import__("datetime").datetime.now( + __import__("datetime").timezone.utc + ), + } + ] + } + + mock_get_client.return_value = mock_client + + result = get_s3_report(days_threshold=90) + + assert result["total_buckets"] == 1 + assert result["new_buckets"] == ["new-bucket"] + assert result["old_buckets"] == [] + + +@patch("app.services.aws_service._get_ec2_client") +def test_get_ec2_report(mock_get_client): + mock_client = MagicMock() + + mock_client.describe_instances.return_value = { + "Reservations": [ + { + "Instances": [ + { + "InstanceId": "i-1234567890", + "State": {"Name": "running"}, + } + ] + } + ] + } + + mock_get_client.return_value = mock_client + + result = get_ec2_report() + + assert result == [ + { + "instance_id": "i-1234567890", + "state": "running", + } + ] + + +def test_invalid_s3_age_threshold(): + with pytest.raises(ValueError): + get_s3_report(days_threshold=-1) \ No newline at end of file diff --git a/capstone/tests/test_logs.py b/capstone/tests/test_logs.py new file mode 100644 index 0000000..df36098 --- /dev/null +++ b/capstone/tests/test_logs.py @@ -0,0 +1,45 @@ +from pathlib import Path + +import pytest + +from app.services.log_service import analyze_log_file, count_log_levels + + +def test_count_log_levels(): + text = """ + INFO Application started + WARNING High memory usage + ERROR Database timeout + """ + + assert count_log_levels(text) == { + "INFO": 1, + "WARNING": 1, + "ERROR": 1, + } + + +def test_count_log_levels_does_not_match_partial_words(): + text = "INFO No errors were found." + + assert count_log_levels(text) == { + "INFO": 1, + "WARNING": 0, + "ERROR": 0, + } + + +def test_analyze_log_file(): + log_file = Path(__file__).parents[1] / "sample_logs" / "app.log" + + result = analyze_log_file(log_file) + + assert result["total_lines"] > 0 + assert result["counts"]["INFO"] > 0 + assert result["counts"]["WARNING"] > 0 + assert result["counts"]["ERROR"] > 0 + + +def test_missing_log_file(): + with pytest.raises(FileNotFoundError): + analyze_log_file("does-not-exist.log") \ No newline at end of file diff --git a/capstone/tests/test_metrics.py b/capstone/tests/test_metrics.py new file mode 100644 index 0000000..6cf4465 --- /dev/null +++ b/capstone/tests/test_metrics.py @@ -0,0 +1,53 @@ +from unittest.mock import patch + +import pytest + +from app.services.metrics_service import get_system_metrics + + +@patch("app.services.metrics_service.psutil.disk_usage") +@patch("app.services.metrics_service.psutil.virtual_memory") +@patch("app.services.metrics_service.psutil.cpu_percent") +def test_get_system_metrics( + mock_cpu_percent, + mock_virtual_memory, + mock_disk_usage, +): + mock_cpu_percent.return_value = 25.0 + mock_virtual_memory.return_value.percent = 50.0 + mock_disk_usage.return_value.percent = 40.0 + + result = get_system_metrics() + + assert result == { + "cpu_percentage": 25.0, + "memory_percentage": 50.0, + "disk_percentage": 40.0, + "cpu_threshold": 85.0, + "system_status": "Healthy", + } + + +@patch("app.services.metrics_service.psutil.disk_usage") +@patch("app.services.metrics_service.psutil.virtual_memory") +@patch("app.services.metrics_service.psutil.cpu_percent") +def test_get_system_metrics_high_cpu( + mock_cpu_percent, + mock_virtual_memory, + mock_disk_usage, +): + mock_cpu_percent.return_value = 95.0 + mock_virtual_memory.return_value.percent = 50.0 + mock_disk_usage.return_value.percent = 40.0 + + result = get_system_metrics() + + assert result["system_status"] == "High CPU" + + +def test_invalid_cpu_threshold(): + with pytest.raises(ValueError): + get_system_metrics(cpu_threshold=101) + + with pytest.raises(ValueError): + get_system_metrics(cpu_threshold=-1) \ No newline at end of file