mancitrus/system-prompts-and-models-of-ai-tools
1
0
Fork 0
mirror of https://github.com/x1xhlol/system-prompts-and-models-of-ai-tools.git synced 2026-08-16 21:24:09 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge 43de50cabe into 0c828e4e89

Browse Source
This commit is contained in:
munagalllm 2026-06-18 21:23:24 +05:30 committed by GitHub
commit 7ec1be10fa
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
35 changed files with 5425 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
.github/workflows/Development.yaml vendored Normal file
View File

@ -0,0 +1,51 @@
name: Bankfeed Deployment to Amazon EKS
on:
push:
branches:
- dev
permissions:
id-token: write
contents: read
jobs:
deploy:
name: Deploy
timeout-minutes: 30
runs-on: ubuntu-latest
environment: Development
steps:
- name: Checkout
uses: actions/checkout@v4.1.7
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: ${{ secrets.AWS_ROLE }}
role-session-name: GitHub_to_AWS_via_FederatedOIDC
aws-region: ${{ secrets.AWS_REGION }}
- name: Login to Amazon ECR Private
id: login-ecr
uses: aws-actions/amazon-ecr-login@v2.0.1
- name: Build, tag, and push image to Amazon ECR
id: build-image
env:
ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
run: |
docker build -t 413144972343.dkr.ecr.us-east-1.amazonaws.com/agents-k8-dev:zfunctions-dev$(echo $GITHUB_SHA | head -c7) -f ./task/Dockerfile ./task
docker push 413144972343.dkr.ecr.us-east-1.amazonaws.com/agents-k8-dev:zfunctions-dev$(echo $GITHUB_SHA | head -c7)
- name: Update deployment file
run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|413144972343.dkr.ecr.us-east-1.amazonaws.com/agents-k8-dev:zfunctions-dev'${TAG}'|' $GITHUB_WORKSPACE/task/zfunctions.yaml
- name: Login AWS EKS on EKS Cluster
run: aws eks update-kubeconfig --name ${{ secrets.CLUSTER_NAME }} --region ${{ secrets.AWS_REGION }}
- name: Deploy to AWS EKS
run: kubectl apply -f $GITHUB_WORKSPACE/task/zfunctions.yaml

49
.github/workflows/Production.yaml vendored Normal file
View File

@ -0,0 +1,49 @@
name: Bankfeed Deployment to Amazon EKS
on:
push:
branches:
- main
permissions:
id-token: write
contents: read
jobs:
deploy:
name: Deploy
timeout-minutes: 30
runs-on: ubuntu-latest
environment: Development
steps:
- name: Checkout
uses: actions/checkout@v4.1.7
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: ${{ secrets.AWS_ROLE }}
role-session-name: GitHub_to_AWS_via_FederatedOIDC
aws-region: ${{ secrets.AWS_REGION }}
- name: Login to Amazon ECR Private
id: login-ecr
uses: aws-actions/amazon-ecr-login@v2.0.1
- name: Build, tag, and push image to Amazon ECR
id: build-image
env:
ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
run: |
docker build -t 413144972343.dkr.ecr.us-east-1.amazonaws.com/agents-k8-dev:zfunctions-prod$(echo $GITHUB_SHA | head -c7) -f ./task/Dockerfile ./task
docker push 413144972343.dkr.ecr.us-east-1.amazonaws.com/agents-k8-dev:zfunctions-prod$(echo $GITHUB_SHA | head -c7)
- name: Update deployment file
run: TAG=$(echo $GITHUB_SHA | head -c7) && sed -i 's|<IMAGE>|413144972343.dkr.ecr.us-east-1.amazonaws.com/agents-k8-dev:zfunctions-prod'${TAG}'|' $GITHUB_WORKSPACE/task/zfunctions-prod.yaml
- name: Login AWS EKS on EKS Cluster
run: aws eks update-kubeconfig --name ${{ secrets.CLUSTER_NAME }} --region ${{ secrets.AWS_REGION }}
- name: Deploy to AWS EKS
run: kubectl apply -f $GITHUB_WORKSPACE/task/zfunctions-prod.yaml

12
.gitignore vendored Normal file
View File

@ -0,0 +1,12 @@
.env
.DS_Store
*.log
# Directories
venv/
# Python
__pycache__/
*.pyc
*.pyo
*.pyd

8
.idea/.gitignore generated vendored Normal file
View File

@ -0,0 +1,8 @@
# Default ignored files
/shelf/
/workspace.xml
# Editor-based HTTP Client requests
/httpRequests/
# Datasource local storage ignored files
/dataSources/
/dataSources.local.xml

45
README.zfunctions.md Normal file
View File

@ -0,0 +1,45 @@
# zFunctions
Use the following instructions to deploy the sample application.
## Requirements
- [Python 3.12](https://www.python.org/downloads/).
Install python 3.12
- [The AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-install.html)
Install the latest version of AWS CLI.
## Setup
Git clone this repo.
```
cd zfunctions
```
Install python virtual environment.
```
pip install virtualenv
python -m venv venv
source venv/bin/activate
pip install -r requirements.txt
```
## Run locally
Configure env file.
```
cp env .env
```
Add credentials
```
cd function/fa_onboarding
python3 lambda_function.py
```
## Deploy
To deploy the application, run ...

1
manifest.json Normal file
View File

@ -0,0 +1 @@
[{"Config":"290e9bd230e751121c05a6b690d98af3fd8cb3071ea42d1050b48a8f0e1b5b95.json","RepoTags":["413144972343.dkr.ecr.us-east-1.amazonaws.com/serverless-fpa-onboarding-dev:fpa-onboardings"],"Layers":["6f1c4d8131b507830019304cf3a00d3c62b091f24c5531d4938c9c90608f8a74/layer.tar","b32028a76a8b26d0d8283ac689268d660d75e71e4d626335cf52099cb362a95d/layer.tar","dbb582ddbebc7074681248e9609c9cfc7dab25d9ec7e8b20c9fc4f71dc2c7183/layer.tar","a32b78699c6eef32c6edaba0ba98ce7c4e46a34e9809e7e2fd02d0779a85fa18/layer.tar","9d21d8586d15f6075ce8af99f8c372bfa600737fb64a3f69df903f02ab876c5e/layer.tar","a27c98cc01a4ead2285115b35470f69da4e4ddf8c3ec34245aa0ef5730ab2871/layer.tar","a0e87182507e5aa8aea1f21d8b3a8abd28f7a1dcef8567327231a42f3c554c8c/layer.tar","04aaf9cfacf692f07def4e0143dc127d3fcf938d604403239c8b9cede93dcf5b/layer.tar","7c9647faac2e8c0dcb5fd729267684ad1d8abe0408caf5e70f49da6011e9c480/layer.tar","04d41fa8c2ef1644a345916b9b1b9c7b3bcf5290f858daa736b0bb3f78ba7e3c/layer.tar","613b72d1e87d3d4611a402c05bb9d1aee94d9810aaa03bdd97d32e4a455c5c82/layer.tar","b2c1d6b777b8b8ef85e9e018ac16e4fe7f8682ff25cfa82ada0a1400d066741e/layer.tar","d7dd4783c680b76ddba436c196ce95f04d984c36658eab4f7a5db3df999f8a3a/layer.tar","ca0cba66bb432dc2cace13dc815bb0f315f917dac8f4c552ed3a1cb1e63b75c2/layer.tar","130a8593783267760265e57ef3a1ea5fa97c689db798578fd92ef3c9c3db6c96/layer.tar","21d71270202e2e82afd5f30586c1811695408b7449708900de9e70a3a4aebd0a/layer.tar","beb36db6ae8ff0dc077729cd974d06de4ba29f5da6924c85c82a89bf9e4eb7f1/layer.tar","e01e882e420c4e2fdf2012dcfa0614d620c2df15ef3c6273c8ff50e9f0d9e0ed/layer.tar","084ab12df9e40bd0d7e466b0c1c2972da82de75e0f9079a18fdca589de4f18a6/layer.tar"]}]

97
serverless.yml Normal file
View File

@ -0,0 +1,97 @@
service: fpa-onboarding
frameworkVersion: '3'
provider:
name: aws
runtime: python3.11
ecr:
images:
fpa-onboardings:
path: ./function/task
platform: linux/amd64
functions:
handler:
image:
name: fpa-onboardings
role: FPAOnboardingServerLessRole
timeout: 300
resources:
Resources:
FPAOnboardingServerLessRole:
Type: AWS::IAM::Role
Properties:
RoleName: fpa-onboarding-serverless-role
AssumeRolePolicyDocument:
Version: '2012-10-17'
Statement:
- Effect: Allow
Principal:
Service:
- lambda.amazonaws.com
Action:
- sts:AssumeRole
Policies:
- PolicyName: FPA-Onboarding-CloudWatchLogsAccess
PolicyDocument:
Version: '2012-10-17'
Statement:
- Effect: Allow
Action:
- logs:CreateLogGroup
- logs:CreateLogStream
- logs:PutLogEvents
Resource: "arn:aws:logs:*:*:*"
- PolicyName: FPA-Onboarding-S3Access
PolicyDocument:
Version: '2012-10-17'
Statement:
- Effect: Allow
Action:
- s3:*
Resource: "*"
- PolicyName: FPA-Onboarding-events
PolicyDocument:
Version: '2012-10-17'
Statement:
- Effect: Allow
Action:
- events:*
Resource: "*"
- PolicyName: FPA-Onboarding-IAMPolicies
PolicyDocument:
Version: '2012-10-17'
Statement:
- Effect: Allow
Action:
- iam:TagRole
Resource: "*"
- PolicyName: FPA-Onboarding-SMAccess
PolicyDocument:
Version: '2012-10-17'
Statement:
- Effect: Allow
Action:
- ssm:GetParameter
- ssm:GetParameters
- ssm:GetParametersByPath
Resource: "*"
- PolicyName: FPA-Onboarding-SQSFullAccess
PolicyDocument:
Version: '2012-10-17'
Statement:
- Effect: Allow
Action:
- sqs:*
Resource: "*"
- PolicyName: FPA-Onboarding-SecretManagerAccess
PolicyDocument:
Version: '2012-10-17'
Statement:
- Effect: Allow
Action:
- secretsmanager:GetSecretValue
- secretsmanager:PutSecretValue
Resource: "*"

33
task/Dockerfile Normal file
View File

@ -0,0 +1,33 @@
FROM python:3.13.2-slim-bullseye
# Install minimal system dependencies and build tools for Python packages
RUN apt-get update && apt-get install -y --no-install-recommends \
build-essential \
gcc \
g++ \
&& rm -rf /var/lib/apt/lists/*
# Set working directory
WORKDIR /app
# Copy only requirements first to leverage Docker cache
COPY requirements.txt .
# Install Python dependencies
RUN pip install --no-cache-dir -r requirements.txt
# Remove build tools to reduce image size
RUN apt-get purge -y --auto-remove \
build-essential \
gcc \
g++
# Copy rest of the app code
COPY . .
# Create a non-root user for security reasons
RUN useradd -m appuser
USER appuser
# Run the Python application
CMD ["python", "sqs_processor.py"]

477
task/backendAPIs.py Normal file
View File

@ -0,0 +1,477 @@
import requests
import os
import json
def get_token():
user_id = os.getenv('USER_ID')
password = os.getenv('PASSWORD')
api_endpoint = os.getenv('API_ENDPOINT')
# print(f"API end point: {api_endpoint}")
url = f'{api_endpoint}/api/v1/token'
credentials = {
"email": user_id,
"password": password
}
headers = {
'Content-Type': 'application/json'
}
try:
response = requests.post(url, json=credentials, headers=headers)
if response.status_code == 200:
data = response.json()
if 'token' in data:
token = data['token']
os.environ["AUTH_TOKEN"] = token
return token
else:
return "Token not found in response."
else:
return f"Failed to get token. Status code: {response.status_code}, Error: {response.text}"
except requests.exceptions.RequestException as e:
return f"An error occurred: {e}"
def get_role_id(entity_id):
api_endpoint = os.getenv('API_ENDPOINT')
auth_token = os.getenv('AUTH_TOKEN')
url = f'{api_endpoint}/api/v1/role_details?entityId={entity_id}'
headers = {
'Content-Type': 'application/json',
'Authorization': auth_token,
}
try:
print(f"*** Calling get role id... for entity {entity_id}")
response = requests.get(url, headers=headers)
response_data = response.json()
if response.status_code == 200 or response.status_code == 201:
role_id = response_data["data"]["_id"]
print(f"Roleid set to {role_id} in the environment.")
return {"data": role_id, "status_code": 200}
else:
print(f"Failed to get data. Status code: {response.status_code}")
print(f"Error: {response.text}")
return {"error": f"Failed to get data. Status code: {response.status_code}",
"status_code": response.status_code, "details": response.text}
except requests.exceptions.RequestException as e:
return {"error": f"An error occurred: {e}", "status_code": 500}
def get_onboarding_status(onboarding_id):
api_endpoint = os.getenv('API_ENDPOINT')
auth_token = os.getenv('AUTH_TOKEN')
# role_id = os.getenv('ROLE_ID')
# url = f'{api_endpoint}/api/v1/entity/onboarding_status?id={onboarding_id}'
# headers = {
# 'Content-Type': 'application/json',
# 'Authorization': auth_token,
# # 'X-Entity-Role': role_id
# }
url = f'{api_endpoint}/api/v1/entity/onboarding_status?id={onboarding_id}'
headers = {
'Content-Type': 'application/json',
'Authorization': auth_token,
# 'X-Entity-Role': role_id
}
try:
response = requests.get(url, headers=headers)
if response.status_code == 200:
# print(response.json())
return {"data": response.json(), "status_code": 200}
else:
print(f"Failed to get data. Status code: {response.status_code}")
print(f"Error: {response.text}")
return {"error": f"Failed to get data. Status code: {response.status_code}",
"status_code": response.status_code, "details": response.text}
except requests.exceptions.RequestException as e:
return {"error": f"An error occurred: {e}", "status_code": 500}
def update_onboarding_status(step_number, status, error_message, summary):
api_endpoint = os.getenv('API_ENDPOINT')
auth_token = os.getenv('AUTH_TOKEN')
role_id = os.getenv('ROLE_ID')
onboarding_id = os.getenv('ONBOARDING_ID')
print(f"Using onboardinid: {onboarding_id}, roleid:{role_id} to update onBoardingStatus.")
url = f"{api_endpoint}/api/v1/entity/onboarding_status?id={onboarding_id}"
headers = {
'Content-Type': 'application/json',
'Authorization': auth_token,
'X-Entity-Role': role_id
}
try:
# -1 to initialize the steps array
if step_number == -1:
data = {
"id": onboarding_id,
"steps": [
{"number": 1, "stepName": "Validate uploaded files", "status": "NOT-STARTED", "errorMessage": None,
"summary": None, "completedAt": None},
{"number": 2, "stepName": "Load fund details", "status": "NOT-STARTED", "errorMessage": None,
"summary": None, "completedAt": None},
{"number": 3, "stepName": "Load partners", "status": "NOT-STARTED", "errorMessage": None,
"summary": None, "completedAt": None},
{"number": 4, "stepName": "Load portfolio (financials) information", "status": "NOT-STARTED",
"errorMessage": None, "summary": None, "completedAt": None},
{"number": 5, "stepName": "Load bank transations", "status": "NOT-STARTED", "errorMessage": None,
"summary": None, "completedAt": None},
{"number": 6, "stepName": "Load journals", "status": "NOT-STARTED", "errorMessage": None,
"summary": None, "completedAt": None},
{"number": 7, "stepName": "Load fund performance", "status": "NOT-STARTED", "errorMessage": None,
"summary": None, "completedAt": None}
]
}
else:
response = get_onboarding_status(onboarding_id)
onboarding_status = response["data"]["data"]
steps = onboarding_status["steps"]
steps[step_number]["status"] = status
steps[step_number]["errorMessage"] = error_message
steps[step_number]["summary"] = summary
data = {
"id": onboarding_id,
"steps": steps
}
response = requests.put(url, headers=headers, data=json.dumps(data))
if response.status_code == 200:
return {"data": response.json(), "status_code": 200}
else:
return {"error": f"Failed to update onboarding status. Status code: {response.status_code}",
"status_code": response.status_code, "details": response.text}
except requests.exceptions.RequestException as e:
return {"error": f"An error occurred: {e}", "status_code": 500}
def get_entity_details(entity_id):
api_endpoint = os.getenv('API_ENDPOINT')
auth_token = os.getenv('AUTH_TOKEN')
url = f'{api_endpoint}/api/v1/entity/entity_details?entityId={entity_id}'
headers = {
'Content-Type': 'application/json',
'Authorization': auth_token
}
response = requests.get(url, headers=headers)
if response.status_code == 200:
# print(response.json())
return response.json()
else:
# Print the status code and error message if the request failed
print(f"Failed to get data. Status code: {response.status_code}")
print(f"Error: {response.text}")
def add_user(user):
api_endpoint = os.getenv('API_ENDPOINT')
auth_token = os.getenv('AUTH_TOKEN')
# role_id = os.getenv('ROLE_ID')
url = f"{api_endpoint}/api/v1/user"
headers = {
'Content-Type': 'application/json',
'Authorization': auth_token,
}
try:
response = requests.post(url, headers=headers, data=json.dumps(user))
if response.status_code == 200:
return {"data": response.json(), "status_code": 200}
else:
return {"error": f"Failed to add user. Status code: {response.status_code}",
"status_code": response.status_code, "details": response.text}
except requests.exceptions.RequestException as e:
return {"error": f"An error occurred: {e}", "status_code": 500}
def add_role(role_payload):
api_endpoint = os.getenv('API_ENDPOINT')
auth_token = os.getenv('AUTH_TOKEN')
role_id = os.getenv('ROLE_ID')
print(f"Role ID going in to add role API: {role_id}")
url = f"{api_endpoint}/api/v1/entity/role/partner"
headers = {
'Content-Type': 'application/json',
'Authorization': auth_token,
'X-Entity-Role': role_id
}
try:
response = requests.post(url, headers=headers, data=json.dumps(role_payload))
if response.status_code == 200 or response.status_code == 201:
return {"data": response.json(), "status_code": 200}
else:
return {"error": f"Failed to add role/user. Status code: {response.status_code}",
"status_code": response.status_code, "details": response.text}
except requests.exceptions.RequestException as e:
return {"error": f"An error occurred: {e}", "status_code": 500}
def update_fund(fund_payload):
api_endpoint = os.getenv('API_ENDPOINT')
auth_token = os.getenv('AUTH_TOKEN')
role_id = os.getenv('ROLE_ID')
print(f"Update fund use role header: {role_id}")
url = f"{api_endpoint}/api/v1/entity/fund_info"
headers = {
'Content-Type': 'application/json',
'Authorization': auth_token,
'X-Entity-Role': role_id
}
try:
response = requests.put(url, headers=headers, data=json.dumps(fund_payload))
if response.status_code == 200 or response.status_code == 201:
return {"data": response.json(), "status_code": 200}
else:
return {"error": f"Failed to add role/user. Status code: {response.status_code}",
"status_code": response.status_code, "details": response.text}
except requests.exceptions.RequestException as e:
return {"error": f"An error occurred: {e}", "status_code": 500}
def add_portfolio_company(payload):
api_endpoint = os.getenv('API_ENDPOINT')
auth_token = os.getenv('AUTH_TOKEN')
role_id = os.getenv('ROLE_ID')
print(f"Update fund use role header: {role_id}")
url = f"{api_endpoint}/api/v1/portfolio-company/single"
headers = {
'Content-Type': 'application/json',
'Authorization': auth_token,
'X-Entity-Role': role_id
}
try:
response = requests.post(url, headers=headers, data=json.dumps(payload))
if response.status_code == 200:
return {"data": response.json(), "status_code": 200}
else:
return {"error": f"Failed to add portfolio company. Status code: {response.status_code}",
"status_code": response.status_code, "details": response.text}
except requests.exceptions.RequestException as e:
return {"error": f"An error occurred: {e}", "status_code": 500}
def add_portfolio_company_investment(payload):
api_endpoint = os.getenv('API_ENDPOINT')
auth_token = os.getenv('AUTH_TOKEN')
role_id = os.getenv('ROLE_ID')
print(f"Update fund use role header: {role_id}")
url = f"{api_endpoint}/api/v1/portfolio-company/investment"
headers = {
'Content-Type': 'application/json',
'Authorization': auth_token,
'X-Entity-Role': role_id
}
try:
response = requests.post(url, headers=headers, data=json.dumps(payload))
if response.status_code == 200:
return {"data": response.json(), "status_code": 200}
else:
return {"error": f"Failed to add portfolio company investment. Status code: {response.status_code}",
"status_code": response.status_code, "details": response.text}
except requests.exceptions.RequestException as e:
return {"error": f"An error occurred: {e}", "status_code": 500}
def add_journal(payload):
api_endpoint = os.getenv('API_ENDPOINT')
auth_token = os.getenv('AUTH_TOKEN')
role_id = os.getenv('ROLE_ID')
print(f"Update fund use role header: {role_id}")
url = f"{api_endpoint}/api/v1/bank/journal"
headers = {
'Content-Type': 'application/json',
'Authorization': auth_token,
'X-Entity-Role': role_id
}
try:
response = requests.post(url, headers=headers, data=json.dumps(payload))
if response.status_code == 200:
return {"data": response.json(), "status_code": 200}
else:
return {"error": f"Failed to add journal. Status code: {response.status_code}",
"status_code": response.status_code, "details": response.text}
except requests.exceptions.RequestException as e:
return {"error": f"An error occurred: {e}", "status_code": 500}
def add_journal_ledger(payload):
print(payload)
api_endpoint = os.getenv('API_ENDPOINT')
auth_token = os.getenv('AUTH_TOKEN')
role_id = os.getenv('ROLE_ID')
print(f"Update fund use role header: {role_id}")
url = f"{api_endpoint}/api/v1/bank/journal-ledger-single"
headers = {
'Content-Type': 'application/json',
'Authorization': auth_token,
'X-Entity-Role': role_id
}
try:
response = requests.post(url, headers=headers, data=json.dumps(payload))
if response.status_code == 200:
return {"data": response.json(), "status_code": 200}
else:
return {"error": f"Failed to add journal ledger. Status code: {response.status_code}",
"status_code": response.status_code, "details": response.text}
except requests.exceptions.RequestException as e:
return {"error": f"An error occurred: {e}", "status_code": 500}
def add_performance_record(payload):
api_endpoint = os.getenv('API_ENDPOINT')
auth_token = os.getenv('AUTH_TOKEN')
role_id = os.getenv('ROLE_ID')
print(f"Update fund use role header: {role_id}")
url = f"{api_endpoint}/api/v1/transaction/name"
headers = {
'Content-Type': 'application/json',
'Authorization': auth_token,
'X-Entity-Role': role_id
}
try:
response = requests.post(url, headers=headers, data=json.dumps(payload))
if response.status_code == 200:
return {"data": response.json(), "status_code": 200}
else:
return {"error": f"Failed to add performance. Status code: {response.status_code}",
"status_code": response.status_code, "details": response.text}
except requests.exceptions.RequestException as e:
return {"error": f"An error occurred: {e}", "status_code": 500}
def add_bank_transaction_records(payload):
api_endpoint = os.getenv('API_ENDPOINT')
auth_token = os.getenv('AUTH_TOKEN')
role_id = os.getenv('ROLE_ID')
print(f"Update fund use role header: {role_id}")
url = f"{api_endpoint}/api/v1/bank"
headers = {
'Content-Type': 'application/json',
'Authorization': auth_token,
'X-Entity-Role': role_id
}
try:
response = requests.post(url, headers=headers, data=json.dumps(payload))
if response.status_code == 200:
return {"data": response.json(), "status_code": 200}
else:
return {"error": f"Failed to add bank transaction. Status code: {response.status_code}",
"status_code": response.status_code, "details": response.text}
except requests.exceptions.RequestException as e:
return {"error": f"An error occurred: {e}", "status_code": 500}
def get_all_role():
api_endpoint = os.getenv('API_ENDPOINT')
url = f"{api_endpoint}/api/v1/entity/role/all_roledata"
headers = {
'Content-Type': 'application/json',
'Authorization': get_token(),
'X-Entity-Role':os.getenv('ROLE_ID')
}
try:
print(f"*** Calling get role id. for all LPAs")
response = requests.get(url, headers=headers)
response_data = response.json()
if response.status_code == 200 or response.status_code == 201:
lpa_role_id_details = response_data["data"]["data"]
# print(f"LPA Roleid {lpa_role_id_details} in the environment.")
return lpa_role_id_details
else:
print(f"Failed to get data. Status code: {response.status_code}")
print(f"Error: {response.text}")
return {"error": f"Failed to get data. Status code: {response.status_code}",
"status_code": response.status_code, "details": response.text}
except requests.exceptions.RequestException as e:
return {"error": f"An error occurred: {e}", "status_code": 500}
def load_partner_data(type,entity,role_id,data):
api_endpoint = os.getenv('API_ENDPOINT')
url = f"{api_endpoint}/api/v1/entity/role/partner_data"
headers = {
'Content-Type': 'application/json',
'Authorization': get_token(),
}
data = {
"entityId":f"{entity}",
"roleId":f"{role_id}",
"miscdata": json.loads(data),
"type":f"{type}"
}
try:
output = requests.post(url, data=json.dumps(data), headers=headers)
if output.status_code == 200 or output.status_code == 201:
print("Success posted the request")
return True
except Exception as e:
print(e)
return False

21
task/file_classifier.py Normal file
View File

@ -0,0 +1,21 @@
import re
import os
from utils.prompts import DOCUMENT_CLASSIFY_PROMPT
from llm_bedrock import model
def document_classifier(record_details):
FINAL_DOCUMENT_CLASSIFY_PROMPT = DOCUMENT_CLASSIFY_PROMPT.format(document=record_details)
json_string = model.invoke(FINAL_DOCUMENT_CLASSIFY_PROMPT).content
print(json_string)
pattern = r'<output>(.*?)</output>'
match = re.search(pattern, json_string, re.DOTALL)
if match:
return match.group(1).strip()
else:
return ""

231
task/file_processor.py Normal file
View File

@ -0,0 +1,231 @@
import os
import requests
from utils.prompts import LPA_STATEMENT_PROMPT, SIDE_LETTER_PROMPT
from backendAPIs import load_partner_data
from llm_bedrock import model
import re
import json
from datetime import datetime, timedelta
from typing import List, Dict
import json
def generate_quarterly_management_fees(
fund_start_date: str,
fund_end_date: str,
fee_schedule: List[Dict[str, float]]
) -> str:
"""
Generates a JSON string of quarterly management fees based on the provided schedule.
:param fund_start_date: Fund start date in 'YYYY-MM-DD' format.
:param fund_end_date: Fund end date in 'YYYY-MM-DD' format.
:param fee_schedule: List of fee changes with 'start_date' and 'annual_fee_percent'.
:return: JSON string representing quarterly management fees.
"""
# Convert string dates to datetime objects
start_date = datetime.strptime(fund_start_date, "%Y-%m-%d")
end_date = datetime.strptime(fund_end_date, "%Y-%m-%d")
# Sort fee_schedule by start_date
fee_schedule_sorted = sorted(fee_schedule, key=lambda x: x['start_date'])
# Create list of fee periods with start and end dates
fee_periods = []
for i, fee in enumerate(fee_schedule_sorted):
period_start = datetime.strptime(fee['start_date'], "%Y-%m-%d")
if i + 1 < len(fee_schedule_sorted):
period_end = datetime.strptime(fee_schedule_sorted[i + 1]['start_date'], "%Y-%m-%d") - timedelta(days=1)
else:
period_end = end_date
# Ensure period doesn't start before fund_start_date
if period_start < start_date:
period_start = start_date
# Ensure period_end doesn't exceed fund_end_date
if period_end > end_date:
period_end = end_date
fee_periods.append({
'start_date': period_start,
'end_date': period_end,
'annual_management_fee_percent': fee['annual_management_fee_percent']
})
# Function to get quarter for a given date
def get_quarter(date):
return (date.month - 1) // 3 + 1
# Initialize a dictionary to hold yearly fees
yearly_fees = {}
current_date = start_date
while current_date <= end_date:
year = current_date.year
if year not in yearly_fees:
yearly_fees[year] = [0.0, 0.0, 0.0, 0.0]
quarter = get_quarter(current_date)
# Find the applicable fee for the current_date
applicable_fee = 0.0
for period in fee_periods:
if period['start_date'] <= current_date <= period['end_date']:
applicable_fee = round(period['annual_management_fee_percent'] / 4, 6) # Convert to decimal per quarter
break
yearly_fees[year][quarter - 1] = applicable_fee
# Move to the next quarter
if quarter == 4:
# Next quarter is Q1 of next year
current_date = datetime(year + 1, 1, 1)
else:
# Next quarter: first month of next quarter
next_quarter_month = 3 * quarter + 1
current_date = datetime(year, next_quarter_month, 1)
# Fill in zero fees before start_date and after end_date
formatted_yearly_fees = {}
for year in range(start_date.year, end_date.year + 1):
if year in yearly_fees:
formatted_yearly_fees[str(year)] = yearly_fees[year]
else:
formatted_yearly_fees[str(year)] = [0.0, 0.0, 0.0, 0.0]
# Convert to JSON string with keys as strings
json_output = json.dumps(formatted_yearly_fees, indent=4)
return json_output
def lpa_amendment(file_data):
FINAL_LPA_PROMPT = LPA_STATEMENT_PROMPT.format(document=file_data)
json_string = model.invoke(FINAL_LPA_PROMPT).content
pattern = r'<output>(.*?)</output>'
match = re.search(pattern, json_string, re.DOTALL)
try:
if match:
final_json_data = match.group(1).strip()
lpa_output = json.loads(final_json_data)
if 'managementFee' in lpa_output and all(key in lpa_output['managementFee'] for key in
['fund_start_date', 'fund_end_date',
'management_fee_schedule']):
formatted_management_fee= generate_quarterly_management_fees( lpa_output['managementFee']['fund_start_date'],lpa_output['managementFee']['fund_end_date'],lpa_output['managementFee']['management_fee_schedule'])
lpa_output['managementFee'] = formatted_management_fee
final_json_data = json.dumps(lpa_output)
print("Final json")
print(final_json_data)
return final_json_data
except Exception as e:
print("")
return None
def create_xml_string(data):
xml_string = '<partners>\n'
for item in data:
xml_string += f' <partner>\n'
xml_string += f' <id>{item.get("_id", "")}</id>\n'
xml_string += f' <partnername>{item.get("partnerExactName", "Not specified")}</partnername>\n'
xml_string += f' </partner>\n'
xml_string += '</partners>'
return xml_string
def side_letter_doc(type,entity_id,file_data,role_data):
output = model.invoke(SIDE_LETTER_PROMPT.format(document=file_data,role_doc=create_xml_string(role_data))).content
pattern = r'<output>(.*?)</output>'
match = re.search(pattern, output, re.DOTALL)
json_data = None
role_id = None
if match:
json_data = match.group(1).strip()
## get id
pattern_id = r'<id>(.*?)</id>'
match = re.search(pattern_id, output, re.DOTALL)
if match:
role_id = match.group(1).strip()
try:
lpa_output = json.loads(json_data)
if 'managementFee' in lpa_output and all(key in lpa_output['managementFee'] for key in
['fund_start_date', 'fund_end_date',
'management_fee_schedule']):
formatted_management_fee = generate_quarterly_management_fees(
lpa_output['managementFee']['fund_start_date'], lpa_output['managementFee']['fund_end_date'],
lpa_output['managementFee']['management_fee_schedule'])
lpa_output['managementFee'] = formatted_management_fee
final_json_data = json.dumps(lpa_output)
print("Final json")
print(final_json_data)
except Exception as e:
print("")
if role_id is None:
return "Role is Not fund"
response = load_partner_data(type,entity_id,role_id,final_json_data)
return response , role_id

487
task/gemini_client.py Normal file
View File

@ -0,0 +1,487 @@
"""
Gemini client — the Gemini side of llm.py's LLM_BACKEND switch.
Activated when LLM_BACKEND=gemini (auto-selected if GCP_API_KEY is set).
Routes Claude-shaped calls through the google-genai SDK with an API key. Exposes:
- GeminiBedrockClient: .invoke_model() drop-in for boto3 bedrock-runtime.
Translates Anthropic-style request body to Gemini and shapes the response
back to the Anthropic JSON the rest of the repo expects.
- GeminiChatModel: langchain ChatBedrock-style .invoke().
- GeminiConverseClient: converse-API wrapper for RLM / tearsheet loops.
- gemini_model_call / gemini_invoke_model: functional helpers used by llm.py.
"""
import base64
import json
import logging
import os
import uuid
logger = logging.getLogger(__name__)
try:
from utils.token_tracker import record_usage as _record_token_usage
except ImportError:
try:
from task.utils.token_tracker import record_usage as _record_token_usage
except ImportError:
def _record_token_usage(_model, _input_tokens, _output_tokens): # type: ignore
pass
GEMINI_MODEL_ID = os.environ.get("GEMINI_MODEL_ID", "gemini-3.5-flash")
GCP_API_KEY = os.environ.get("GCP_API_KEY") or os.environ.get("GEMINI_API_KEY", "")
_client_init_logged = False
_gemini_client_singleton = None
def _get_gemini_client():
"""Return a google-genai Client authenticated with GCP_API_KEY.
Lazily constructed and cached: the genai client (and the GCP_API_KEY
requirement) is only resolved on the FIRST LLM call, not at module import.
This keeps a missing GCP_API_KEY from crashing the whole worker at boot —
it surfaces only on the specific request that needs the model.
"""
global _gemini_client_singleton
if _gemini_client_singleton is not None:
return _gemini_client_singleton
from google import genai
if not GCP_API_KEY:
raise RuntimeError(
"GCP_API_KEY (or GEMINI_API_KEY) is not set. "
"Either set it or switch LLM_BACKEND."
)
global _client_init_logged
if not _client_init_logged:
_client_init_logged = True
logger.info("[gemini_client] init: model=%s (api-key auth)", GEMINI_MODEL_ID)
print(f"[gemini_client] init: model={GEMINI_MODEL_ID} (api-key auth)", flush=True)
_gemini_client_singleton = genai.Client(api_key=GCP_API_KEY)
return _gemini_client_singleton
def _to_gemini_contents(messages: list[dict]):
"""Convert Anthropic-style messages to google-genai Content list.
Anthropic role "assistant" maps to Gemini role "model". Image blocks
in Anthropic format ({"type": "image", "source": {"type": "base64",
"media_type": ..., "data": ...}}) become inline Parts.
"""
from google.genai import types
contents = []
for msg in messages:
role = "model" if msg.get("role") == "assistant" else "user"
content = msg.get("content")
parts = []
if isinstance(content, str):
parts.append(types.Part.from_text(text=content))
elif isinstance(content, list):
for block in content:
btype = block.get("type")
if btype == "text":
parts.append(types.Part.from_text(text=block.get("text", "")))
elif btype == "image":
src = block.get("source", {}) or {}
if src.get("type") == "base64":
parts.append(types.Part.from_bytes(
data=base64.b64decode(src.get("data", "")),
mime_type=src.get("media_type", "image/png"),
))
else:
# Tearsheet/Bedrock converse-style block — {"text": ...} or {"image": ...}.
if "text" in block:
parts.append(types.Part.from_text(text=block["text"]))
elif "image" in block:
img = block["image"]
fmt = img.get("format", "png")
data = img.get("source", {}).get("bytes", b"")
parts.append(types.Part.from_bytes(
data=data, mime_type=f"image/{fmt}",
))
else:
parts.append(types.Part.from_text(text=str(content)))
contents.append(types.Content(role=role, parts=parts))
return contents
def _build_generate_config(*, system=None, max_tokens=4096, temperature=0,
use_thinking=False, thinking_budget=None,
json_mode=False, response_schema=None,
use_search=False):
"""Assemble a GenerateContentConfig with optional thinking + system prompt.
json_mode=True forces structured JSON output via the Gemini decoder
(`response_mime_type="application/json"`) — the model cannot emit
prose / markdown / table rows even if the prompt is sloppy.
response_schema (optional) binds the output to a specific shape. Pass
a Python type (`list[str]`, a TypedDict, a pydantic BaseModel) or a
`types.Schema` dict. Implies json_mode. The decoder will refuse to
emit anything that doesn't conform.
use_search=True attaches Gemini's built-in Google Search grounding tool
so the model searches the live web before answering. OFF by default —
callers opt in per request (it adds latency + search cost and makes
output web-dependent). Verified on gemini-2.5-pro to coexist
with json_mode / response_schema and with thinking, so it composes with
the other flags here.
"""
from google.genai import types
cfg_kwargs = {
"temperature": temperature,
"max_output_tokens": max_tokens,
}
if system:
cfg_kwargs["system_instruction"] = system
if use_thinking or (thinking_budget and thinking_budget > 0):
budget = thinking_budget if (thinking_budget and thinking_budget > 0) else 5000
cfg_kwargs["thinking_config"] = types.ThinkingConfig(thinking_budget=budget)
if json_mode or response_schema is not None:
cfg_kwargs["response_mime_type"] = "application/json"
if response_schema is not None:
cfg_kwargs["response_schema"] = response_schema
if use_search:
# Gemini 2.0+ grounding uses GoogleSearch(); serializes to the same
# {"google_search": {}} tool the vendor-research REST path sends.
cfg_kwargs["tools"] = [types.Tool(google_search=types.GoogleSearch())]
return types.GenerateContentConfig(**cfg_kwargs)
def _extract_text(response) -> str:
"""Pull the concatenated text from a google-genai GenerateContentResponse."""
if getattr(response, "text", None):
return response.text
out = []
for cand in getattr(response, "candidates", []) or []:
content = getattr(cand, "content", None)
if not content:
continue
for part in getattr(content, "parts", []) or []:
if getattr(part, "text", None):
out.append(part.text)
return "".join(out)
def _extract_usage(response) -> tuple[int, int]:
"""Pull token counts from response.usage_metadata per Gemini API docs.
Documented fields (https://ai.google.dev/gemini-api/docs/tokens):
- prompt_token_count — input tokens
- candidates_token_count — output tokens (excluding thinking)
- thoughts_token_count — thinking tokens (billed as OUTPUT per
https://ai.google.dev/gemini-api/docs/pricing
"Output price (including thinking tokens)")
- cached_content_token_count, total_token_count — informational
Returns (input_tokens, output_tokens) using:
input = prompt_token_count
output = candidates_token_count + thoughts_token_count
"""
um = getattr(response, "usage_metadata", None)
if um is None:
return 0, 0
in_toks = int(getattr(um, "prompt_token_count", 0) or 0)
out_toks = (
int(getattr(um, "candidates_token_count", 0) or 0)
+ int(getattr(um, "thoughts_token_count", 0) or 0)
)
return in_toks, out_toks
def _record_usage(response, model_id: str) -> tuple[int, int]:
"""Manual token tracking — read documented fields, record once."""
in_toks, out_toks = _extract_usage(response)
if in_toks or out_toks:
_record_token_usage(model_id, in_toks, out_toks)
return in_toks, out_toks
def _stream_generate(client, *, model: str, contents, config):
"""Run generate_content_stream and return (full_text, last_chunk).
Streaming avoids the 10-minute timeout on long requests. The last chunk
carries the cumulative ``usage_metadata`` for token tracking.
"""
text_parts: list[str] = []
last_chunk = None
for chunk in client.models.generate_content_stream(
model=model, contents=contents, config=config,
):
last_chunk = chunk
chunk_text = _extract_text(chunk)
if chunk_text:
text_parts.append(chunk_text)
return "".join(text_parts), last_chunk
class _StreamBody:
"""Mimic boto3 StreamingBody so callers can do response['body'].read()."""
def __init__(self, data: bytes):
self._data = data
def read(self):
return self._data
class GeminiBedrockClient:
"""Drop-in replacement for boto3 bedrock-runtime client.
Translates an Anthropic-shaped request body to Gemini, then shapes the
response back to Anthropic JSON so the rest of the repo (which parses
response_body['content'][i]['text'] / 'usage') keeps working.
Token tracking — pure shim: returns the real ``usage`` in the response
and does NOT record itself. The caller
parses ``response_body['usage']`` and records via the documented
``record_usage`` API (``task/utils/token_tracker.py:218``). Token counts
are computed from the documented ``usage_metadata`` fields per the
Gemini docs (``_extract_usage``).
"""
def __init__(self):
# Client built lazily on first invoke_model() so a missing GCP_API_KEY
# does not crash the worker at import/construction time.
self._model = GEMINI_MODEL_ID
@property
def _client(self):
return _get_gemini_client()
def invoke_model(self, modelId: str, body: str, **_kwargs):
logger.info("[gemini_client.GeminiBedrockClient.invoke_model] model=%s (caller passed modelId=%s)", self._model, modelId)
request = json.loads(body) if isinstance(body, str) else body
messages = request.get("messages", [])
max_tokens = request.get("max_tokens", 16384)
temperature = request.get("temperature", 0)
system = request.get("system")
thinking = request.get("thinking") or {}
use_thinking = thinking.get("type") == "enabled"
budget = thinking.get("budget_tokens") if use_thinking else None
# Optional Google Search grounding — set "use_search": true in the
# request body (non-Anthropic field; absent → off, behavior unchanged).
use_search = bool(request.get("use_search", False))
# Unlike Anthropic, Gemini accepts thinking WITH temperature=0 — keep
# the caller's temperature (default 0) so thinking calls stay
# deterministic. Never force temperature=1 here.
contents = _to_gemini_contents(messages)
config = _build_generate_config(
system=system, max_tokens=max_tokens, temperature=temperature,
use_thinking=use_thinking, thinking_budget=budget,
use_search=use_search,
)
text, last_chunk = _stream_generate(
self._client, model=self._model, contents=contents, config=config,
)
in_toks, out_toks = _extract_usage(last_chunk) if last_chunk else (0, 0)
response_dict = {
"id": f"gemini-{uuid.uuid4().hex[:24]}",
"type": "message",
"role": "assistant",
"content": [{"type": "text", "text": text}],
"model": self._model,
"stop_reason": "end_turn",
"usage": {
"input_tokens": in_toks,
"output_tokens": out_toks,
},
}
return {"body": _StreamBody(json.dumps(response_dict).encode("utf-8"))}
class _InvokeResult:
def __init__(self, content: str):
self.content = content
class GeminiChatModel:
"""Drop-in replacement for LangChain ChatBedrock.
The ``model_id`` argument is accepted for signature compatibility with
callers that hardcode a Bedrock id (e.g. ``us.anthropic.claude-sonnet-4-6``)
but is **ignored** — under ``LLM_BACKEND=gemini`` we always call
``GEMINI_MODEL_ID``.
"""
def __init__(self, model_id: str = None, model_kwargs: dict = None):
del model_id # accepted for signature compat; we always use GEMINI_MODEL_ID
# Client built lazily on first invoke() so a missing GCP_API_KEY does
# not crash the worker at import/construction time.
self._model = GEMINI_MODEL_ID
self._kwargs = model_kwargs or {}
@property
def _client(self):
return _get_gemini_client()
def invoke(self, prompt):
logger.info("[gemini_client.GeminiChatModel.invoke] model=%s", self._model)
if isinstance(prompt, str):
messages = [{"role": "user", "content": prompt}]
else:
messages = prompt
config = _build_generate_config(
max_tokens=self._kwargs.get("max_tokens", 16384),
temperature=self._kwargs.get("temperature", 0),
use_search=bool(self._kwargs.get("use_search", False)),
)
text, last_chunk = _stream_generate(
self._client, model=self._model,
contents=_to_gemini_contents(messages), config=config,
)
if last_chunk is not None:
_record_usage(last_chunk, self._model)
return _InvokeResult(text)
class GeminiConverseClient:
"""Drop-in replacement for pdf_to_xlsx / tearsheets BedrockClient.
The ``model_id`` and ``region`` arguments are accepted for signature
compatibility (RLMBedrockClient callers hardcode Bedrock model ids and
AWS regions) but are **ignored** — under ``LLM_BACKEND=gemini`` we always
call ``GEMINI_MODEL_ID``.
"""
def __init__(
self,
model_id: str = None,
region: str = None,
max_tokens: int = 16384,
thinking_budget: int = 4096,
use_search: bool = False,
):
del model_id, region # accepted for signature compat; always GEMINI_MODEL_ID
self.model_id = GEMINI_MODEL_ID
self.max_tokens = max_tokens
self.thinking_budget = thinking_budget if (thinking_budget and thinking_budget > 0) else 0
self.use_search = use_search
# Client built lazily on first completion() so a missing GCP_API_KEY
# does not crash the worker at import/construction time.
self.total_input_tokens = 0
self.total_output_tokens = 0
self.call_count = 0
@property
def _client(self):
return _get_gemini_client()
def _generate(self, messages: list[dict], system: str | None) -> str:
logger.info(
"[gemini_client.GeminiConverseClient] model=%s thinking=%s call#%d",
self.model_id, bool(self.thinking_budget), self.call_count + 1,
)
contents = _to_gemini_contents(messages)
config = _build_generate_config(
system=system,
max_tokens=self.max_tokens,
temperature=0, # Gemini allows thinking at temp 0 — deterministic
use_thinking=bool(self.thinking_budget),
thinking_budget=self.thinking_budget or None,
use_search=self.use_search,
)
text, last_chunk = _stream_generate(
self._client, model=self.model_id, contents=contents, config=config,
)
if last_chunk is not None:
in_toks, out_toks = _record_usage(last_chunk, self.model_id)
self.total_input_tokens += in_toks
self.total_output_tokens += out_toks
self.call_count += 1
return text
def completion(self, messages: list[dict], system: str | None = None) -> str:
return self._generate(messages, system)
def completion_with_image(
self,
image_bytes: bytes,
mime_type: str,
prompt: str,
system: str | None = None,
) -> str:
media_type = mime_type if "/" in mime_type else f"image/{mime_type}"
b64 = base64.standard_b64encode(image_bytes).decode("utf-8")
messages = [{
"role": "user",
"content": [
{"type": "image", "source": {
"type": "base64", "media_type": media_type, "data": b64,
}},
{"type": "text", "text": prompt},
],
}]
return self._generate(messages, system)
def usage_summary(self) -> str:
return (
f"Calls: {self.call_count} | "
f"Input: {self.total_input_tokens:,} | "
f"Output: {self.total_output_tokens:,}"
)
def gemini_model_call(prompt, max_tokens=16384, use_thinking=False, system=None,
json_mode=False, response_schema=None, use_search=False):
"""Single-shot Gemini call. Returns the text response.
use_search=True grounds the call with Google Search (off by default)."""
logger.info("[gemini_client.gemini_model_call] model=%s thinking=%s json_mode=%s schema=%s search=%s", GEMINI_MODEL_ID, use_thinking, json_mode, response_schema is not None, use_search)
client = _get_gemini_client()
config = _build_generate_config(
system=system, max_tokens=max_tokens,
temperature=0, # Gemini allows thinking at temp 0 — deterministic
use_thinking=use_thinking,
json_mode=json_mode,
response_schema=response_schema,
use_search=use_search,
)
contents = _to_gemini_contents([{"role": "user", "content": prompt}])
try:
text, last_chunk = _stream_generate(
client, model=GEMINI_MODEL_ID, contents=contents, config=config,
)
if last_chunk is not None:
_record_usage(last_chunk, GEMINI_MODEL_ID)
return text
except Exception as e:
logger.error(f"Gemini model call failed: {e}")
raise
def gemini_invoke_model(messages, system=None, max_tokens=16384, temperature=0,
use_thinking=False, use_search=False):
"""Multi-turn Gemini call. Returns the text response.
use_search=True grounds the call with Google Search (off by default)."""
logger.info("[gemini_client.gemini_invoke_model] model=%s msgs=%d thinking=%s search=%s", GEMINI_MODEL_ID, len(messages), use_thinking, use_search)
client = _get_gemini_client()
config = _build_generate_config(
system=system, max_tokens=max_tokens,
temperature=temperature, # Gemini allows thinking at temp 0 — no forcing
use_thinking=use_thinking,
use_search=use_search,
)
try:
text, last_chunk = _stream_generate(
client, model=GEMINI_MODEL_ID,
contents=_to_gemini_contents(messages), config=config,
)
if last_chunk is not None:
_record_usage(last_chunk, GEMINI_MODEL_ID)
return text
except Exception as e:
logger.error(f"Gemini invoke_model failed: {e}")
raise

47
task/initOnboarding.py Normal file
View File

@ -0,0 +1,47 @@
import os
import json
from backendAPIs import (
get_token,
get_role_id,
update_onboarding_status,
get_onboarding_status
)
def initialize_onboarding():
print(f"Getting Auth token...")
token = get_token()
# print(f"Token {token}")
onboarding_id = os.getenv('ONBOARDING_ID')
print(f"Get onBoardingStatus for {onboarding_id}")
response = get_onboarding_status(onboarding_id)
if 'error' in response:
print(f"Failed to fetch entityid: Error: {response['error']}")
print(f"Status Code: {response['status_code']}")
return False
onboarding_status = response['data']['data']
print(f"Onboading status {onboarding_status}")
entity_id = onboarding_status['entityId']
os.environ["ENTITY_ID"] = entity_id
print(f"Entityid set to {entity_id} in the environment")
print(f"Getting role id...")
response = get_role_id(entity_id)
print(response)
if 'error' in response:
print(f"Failed to fetch role id: Error: {response['error']}")
print(f"Status Code: {response['status_code']}")
return False
role_id = response['data']
os.environ["ROLE_ID"] = role_id
print(f"Roleid set to {role_id} in the environment")
response = update_onboarding_status(-1, "", "", "")
if 'error' in response:
print(f"Failed to fetch onboarding status: Error: {response['error']}")
print(f"Status Code: {response['status_code']}")
return False

155
task/lambda_function.py Normal file
View File

@ -0,0 +1,155 @@
import json
import os
from initOnboarding import initialize_onboarding
from validatedUploadedFiles import run_file_validation
from onboardFundsDummy import onboardFunds
from onboardPartners import process_all_partners
from onboardBankTransactions import process_bank_transactions
from onboardFinancials import process_financials
from onboardJournals import process_journals
from onboardPerformance import process_fund_performance
from backendAPIs import get_all_role
from file_processor import lpa_amendment , side_letter_doc
from backendAPIs import load_partner_data
def lambda_handler(event):
# Extract the 'id'
onboarding_id = json.loads(event)['id']
print(f"Received onboarding id from Q: {onboarding_id}")
if not onboarding_id:
return {
'statusCode': 400,
'body': json.dumps('Missing onboarding_id')
}
os.environ["ONBOARDING_ID"] = onboarding_id
print(f"Onboarding id set to {onboarding_id} in the environment")
print(f"Received onboarding id from Q: {onboarding_id}")
if not onboarding_id:
return {
'statusCode': 400,
'body': json.dumps('Missing onboarding_id')
}
os.environ["ONBOARDING_ID"] = onboarding_id
print(f"Onboarding id set to {onboarding_id} in the environment")
print("0. Initialising...")
response = initialize_onboarding()
if response is False:
print("Initialization failed, terminating onboarding...")
return False
print("1. Validating input files...")
found_files ,pdf_files = run_file_validation()
if not isinstance(found_files, dict):
print(found_files)
error_message = found_files
return {
'statusCode': 400,
'body': json.dumps(error_message)
}
print("2. Running fund onboarding...")
onboardFunds(found_files["partner"])
print("3. Running partner onboarding...")
process_all_partners(found_files["partner"])
print("4. Running financial onboarding...")
process_financials(found_files["financials"]) # use Roll Forward ->column B->Security type and from rollforward need to make.
print("5. Running bank transactions onboarding...")
process_bank_transactions(found_files["bankTransactions"])
print("6. Running journal onboarding...")
process_journals(found_files["journals"]) # issue with missing account types
print("7. Running fund performance onboarding...")
process_fund_performance(found_files["fund_performance"]) #
# start processing PDF files related to GP based on the type of the document
#
# print("Start processing the PDF files related to GP")
#
# # give list of the files to classifier function to classify them
#
# role_data = get_all_role()
#
# print("GET ALL role data")
#
# print(role_data)
#
# classified_pdf_files = pdf_classifier(pdf_files)
#
# print("Processing the LPA and its Amendments")
#
# side_letter_files = set()
#
# for file_category , file_names in classified_pdf_files.items():
#
# for file_name in file_names:
#
# output_json = None
#
# pdf_string = get_pdf_details(file_name)
#
# print(f"Started processing the file : {file_name}")
#
# for role_id in role_data:
#
# print(f"Start processing the file :{file_name} for the role id :{role_id['_id']}")
#
#
# if file_category.strip() == "LPA(Limited Partner Agreement)":
#
# if output_json is None:
# output_json = lpa_amendment(pdf_string)
#
# response = load_partner_data("LPA", os.environ["ENTITY_ID"], role_id['_id'], output_json)
#
# if not response:
# print(
# f"Issue encouneted in file processing of {file_name} for the role id : {role_id['_id']}")
# else:
# print(f"Successfully processed : {file_name} for the role id : {role_id['_id']}")
#
# if file_category.strip() == "LPA-Amendment(Limited Partner Agreement-Amendment)":
# #
# if output_json is None:
# output_json = lpa_amendment(pdf_string)
#
# response = load_partner_data("Amendment", os.environ["ENTITY_ID"], role_id['_id'], output_json)
#
# if not response:
# print(
# f"Issue encouneted in file processing of {file_name} for the role id : {role_id['_id']}")
# else:
# print(f"Successfully processed : {file_name} for the role id : {role_id['_id']}")
#
# if file_category.strip() == "Side Letter":
# side_letter_files = file_names
#
# ### lets start the file processing for side letter because its different for each LP and each role id need to seperated
#
# for side_letter_file in side_letter_files:
#
# print(side_letter_file)
#
# pdf_string = get_pdf_details(side_letter_file)
#
# status , role_id = side_letter_doc("Side Letter",os.environ["ENTITY_ID"],pdf_string,role_data)
#
# if not status:
# print(f"Issue encouneted in file processing of side letter {side_letter_file} and related role id : {role_id}")
#
# else:
# print(f"Side letter file : {side_letter_file} was processed successfully and related role id : {role_id}")
return

130
task/llm.py Normal file
View File

@ -0,0 +1,130 @@
"""
Unified LLM interface — Gemini-only build for the Carta onboarding service.
This service runs exclusively on Google Gemini (google-genai SDK, API-key auth
via the GCP_API_KEY env var). The AWS Bedrock and GCP Vertex backends that the
shared lambdaLLM module supports have been removed here: there is a single
provider and a single code path.
All call sites import from this module so the provider stays centralized:
from llm import get_chat_model # chat model with .invoke()
from llm import get_llm_client # low-level client with .invoke_model()
from llm import model_call, invoke_messages, get_converse_client
Retry strategy: on rate-limit / quota errors, wait a fixed interval until the
provider's limit window resets and retry until the call succeeds (no
exponential backoff). Non-retryable errors are raised immediately.
"""
import logging
import os
import time
logger = logging.getLogger(__name__)
# Backend is fixed to Gemini. Exposed as a constant (and written back to the
# environment) so any code or dependency that still reads LLM_BACKEND sees the
# only supported value.
LLM_BACKEND = "gemini"
os.environ["LLM_BACKEND"] = LLM_BACKEND
# Model id is config-driven so ops can flip Gemini models without code changes.
GEMINI_MODEL_ID = os.environ.get("GEMINI_MODEL_ID", "gemini-3.5-flash")
ACTIVE_MODEL_ID = GEMINI_MODEL_ID
logger.info("[llm] backend=gemini | active_model=%s", ACTIVE_MODEL_ID)
print(f"[llm] backend=gemini | active_model={ACTIVE_MODEL_ID}", flush=True)
# Retry config. LLM_RETRY_WAIT is the fixed wait (seconds) between retries;
# LLM_MAX_RETRIES is a safety cap (default 1000 ~= 16h at 60s wait).
LLM_RETRY_WAIT = float(os.environ.get("LLM_RETRY_WAIT", "60.0"))
LLM_MAX_RETRIES = int(os.environ.get("LLM_MAX_RETRIES", "1000"))
def _is_rate_limit(exc):
"""Return True for throttling / rate-limit / quota errors from google-genai."""
msg = str(exc).lower()
if "google.genai" in type(exc).__module__:
code = getattr(exc, "code", None)
if isinstance(code, int) and code in {408, 429, 500, 502, 503, 504}:
return True
for token in ("throttl", "rate limit", "too many requests", "quota",
"overloaded", "resource_exhausted", "429", "503",
"service unavailable"):
if token in msg:
return True
return False
def _with_retry(fn, *args, **kwargs):
"""Invoke fn and wait out provider rate-limit windows until it succeeds."""
for attempt in range(LLM_MAX_RETRIES + 1):
try:
return fn(*args, **kwargs)
except Exception as e: # noqa: BLE001
if not _is_rate_limit(e) or attempt >= LLM_MAX_RETRIES:
raise
logger.warning(
"LLM rate-limited (%s) on attempt %d; waiting %.0fs for limit window to reset: %s",
type(e).__name__, attempt + 1, LLM_RETRY_WAIT, e,
)
time.sleep(LLM_RETRY_WAIT)
raise RuntimeError("LLM retry loop exhausted without success")
def _gemini():
"""Import the gemini_client module (top-level or task-package layout)."""
try:
import gemini_client as gc
except ImportError:
from task import gemini_client as gc # type: ignore
return gc
def get_llm_client():
"""Return a low-level client exposing .invoke_model(), backed by
google-genai (GeminiBedrockClient)."""
return _gemini().GeminiBedrockClient()
def model_call(prompt, max_tokens=16384, use_thinking=False, system=None,
json_mode=False, response_schema=None, use_search=False):
"""Single-shot Gemini call. Returns the text response. Retries on throttling."""
return _with_retry(
_gemini().gemini_model_call, prompt,
max_tokens=max_tokens, use_thinking=use_thinking, system=system,
json_mode=json_mode, response_schema=response_schema, use_search=use_search,
)
def invoke_messages(messages, system=None, max_tokens=16384, temperature=0,
use_thinking=False, use_search=False):
"""Multi-turn Gemini call. Returns the text response. Retries on throttling."""
return _with_retry(
_gemini().gemini_invoke_model, messages=messages, system=system,
max_tokens=max_tokens, temperature=temperature,
use_thinking=use_thinking, use_search=use_search,
)
def get_chat_model(model_id=None, temperature=0, max_tokens=16384,
model_kwargs=None, use_search=False):
"""Return a chat model with .invoke() (GeminiChatModel).
model_id is accepted for call-site compatibility but ignored — the active
model is always GEMINI_MODEL_ID. GeminiChatModel records token usage
internally."""
kwargs = dict(model_kwargs) if model_kwargs else {}
kwargs.setdefault("temperature", temperature)
kwargs.setdefault("max_tokens", max_tokens)
if use_search:
kwargs["use_search"] = True
return _gemini().GeminiChatModel(model_id=model_id or GEMINI_MODEL_ID,
model_kwargs=kwargs)
def get_converse_client(max_tokens=16384, thinking_budget=1024, use_search=False):
"""Return a converse-style Gemini client (completion / completion_with_image)."""
return _gemini().GeminiConverseClient(
max_tokens=max_tokens, thinking_budget=thinking_budget, use_search=use_search,
)

13
task/llm_bedrock.py Normal file
View File

@ -0,0 +1,13 @@
"""Legacy module name kept for import compatibility.
Bedrock has been removed from this service — `model` is now the Gemini-backed
chat model from the unified Gemini-only interface in task/llm.py. Existing
`from llm_bedrock import model` call sites keep working unchanged.
"""
try:
from llm import get_chat_model
except ImportError:
from task.llm import get_chat_model # type: ignore
model_parameter = {"temperature": 0, "max_tokens": 4096}
model = get_chat_model(model_kwargs=model_parameter)

136
task/onboardBankTransactions.py Normal file
View File

@ -0,0 +1,136 @@
import pandas as pd
import json
import os
from s3Ops import read_file_from_s3
from backendAPIs import (
update_onboarding_status,
add_bank_transaction_records
)
def xlsx_to_df(xlsx_file, sheet_name):
df = pd.read_excel(xlsx_file, sheet_name = sheet_name)
df.columns = df.iloc[2]
df = df.drop(df.index[:3])
df = df.reset_index(drop=True)
return df
# TODO: add the remaining asset type mapping
def get_security_type(asset_type):
asset_to_security_type = {
"Common stock":"COMMON",
"Preferred stock":"PREFERRED",
"Warrants":"WARRANTS",
"Convertible promissory note/SAFEs": "POST MONEY SAFE"
}
for asset, security_type in asset_to_security_type.items():
if asset_type.lower() in asset.lower(): # Case-insensitive match
return security_type
return ""
def process_bank_transaction_records(df):
"""
Process and insert users from an Excel file into MongoDB.
Args:
file_path (str): Path to the Excel file.
"""
entity_id = os.getenv('ENTITY_ID')
fund_id = os.getenv('FUND_ID')
# Filter out rows with invalid Journal IDs
# df = df[~df['Journal ID'].isin(['Journal ID'])]
print(df.head())
transactions = []
# Iterate through the DataFrame and prepare partner records
for index, row in df.iterrows():
tdate = row['Date']
transaction_record = {
"fundId": fund_id,
"date": tdate.strftime('%m/%d/%Y'),
"memo": row['Name'] if pd.notna(row['Name']) else row['Memo'],
"credit": row['Credit'] if pd.notna(row['Credit']) else 0.0,
"debit": row['Debit'] if pd.notna(row['Debit']) else 0.0,
"balance": row['Balance'] if pd.notna(row['Balance']) else 0.0
}
response = add_bank_transaction_records(transaction_record)
print(f"Adding transaction {index}:\n {transaction_record}\n\n")
if 'error' in response:
print(f"Failed to fetch onboarding status: Error: {response['error']}")
print(f"Status Code: {response['status_code']}")
break # Process next company investment record...
transactions.append(transaction_record)
# Test with smaller set of records
# if index == 1:
# break
return transactions
def process_bank_transactions(file_path):
# Process bank transactions
step_number = 5
step_number -= 1
success_message = ""
error_message = ""
status = ""
bucket_name = os.getenv('S3_UPLOAD_BUCKET_NAME')
print(f"Processing file: {file_path}")
response = update_onboarding_status(step_number, "IN-PROGRESS", "", "")
if 'error' in response:
print(f"Failed to fetch onboarding status: Error: {response['error']}")
print(f"Status Code: {response['status_code']}")
return False
journal_excel_file = read_file_from_s3(bucket_name, file_path)
sheet_name = 'Bank Transactions Report'
transaction_df = xlsx_to_df(journal_excel_file, sheet_name)
print(transaction_df.head())
transactions = process_bank_transaction_records(transaction_df)
if transactions:
item_count = len(transactions)
success_message = f"Onboarded {item_count} bank transactions."
status = "COMPLETE"
else:
error_message = "No transactions were onboaerded."
status = "FAILED"
response = update_onboarding_status(step_number, status, error_message, success_message)
if 'error' in response:
print(f"Failed to fetch onboarding status: Error: {response['error']}")
print(f"Status Code: {response['status_code']}")
return False
# print(f"Journals list:\n {transactions}")
return True
if __name__ == "__main__":
from dotenv import load_dotenv
load_dotenv()
onboardingId = "66ea163564e2f97a059160ef"
os.environ["ONBOARDING_ID"] = onboardingId
os.environ["FUND_ID"] = "66c5e6d89ecbf552a05b84fc"
from initOnboarding import initialize_onboarding
initialize_onboarding()
found_files = {'lpa': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/0-Please_DocuSign_CerraCap_II_LP_Limited_Partn.pdf', 'partner': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/0-1-cerracap-ii-lp_2024-07_09_short_partner.xlsx', 'financials': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/2-3-cerracap-ii-lp_2024-08-26_financials.xlsx', 'bankTransactions': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/4-CerraCap_II__LP_bank_transactions_2016-01-01-2024-07-03.xlsx', 'journals': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/5-cerracap-ii-lp_2024-07-09_journals-export.xlsx', 'fund_performance': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/6-cerracap-ii-lp_2024-07-09_fund-performance-report.xlsx'}
financials_excel_file_path = found_files["bankTransactions"]
process_bank_transactions(financials_excel_file_path)

243
task/onboardFinancials.py Normal file
View File

@ -0,0 +1,243 @@
import pandas as pd
import json
import os
import re
from s3Ops import read_file_from_s3
from utils.prompts import ZIVE_SECURITY_TYPE_PROMPT
from backendAPIs import (
update_onboarding_status,
add_portfolio_company,
add_portfolio_company_investment
)
# LLM via the unified Gemini-only backend (see task/llm.py).
from llm import get_chat_model
model_parameter = {"temperature": 0, "max_tokens": 16384}
llm_bedrock = get_chat_model(model_kwargs=model_parameter)
def xlsx_to_df(xlsx_file, sheet_name):
# Read the Excel file, skipping the header rows
df = pd.read_excel(xlsx_file, sheet_name=sheet_name, header=5, na_values=[''])
print(df.columns)
investments = {}
current_investment = None
# Iterate through the rows
for index, row in df.iterrows():
if pd.notna(row['Investment legal name']):
# Start a new investment
current_investment = row['Investment legal name']
print(current_investment)
investments[current_investment] = {'securities': [], 'security_count': 0}
elif pd.notna(row['Security type']) and current_investment is not None:
security_type = row['Security type'].strip().lower()
if security_type != 'totals' and security_type != 'grand totals':
# Increment the security count for this investment
investments[current_investment]['security_count'] += 1
# Add security details to the current investment
security = {
'type': row['Security type'],
'investment_date': row['Investment date'],
'investment_number': investments[current_investment]['security_count']
}
investments[current_investment]['securities'].append(security)
return investments
# TODO: add the remaining asset type mapping
# def get_security_type(asset_type):
# asset_to_security_type = {
# "Common stock":"COMMON",
# "Preferred stock":"PREFERRED",
# "Warrants":"WARRANTS",
# "Convertible promissory note/SAFEs": "POST MONEY SAFE"
# }
#
# for asset, security_type in asset_to_security_type.items():
# if asset_type.lower() in asset.lower(): # Case-insensitive match
# return security_type
#
# return ""
def get_security_type(asset_type):
print(asset_type)
print("Calling LLM to get the security type")
FINAL_ZIVE_SECURITY_TYPE_PROMPT = ZIVE_SECURITY_TYPE_PROMPT.format(security_type=asset_type)
output = llm_bedrock.invoke(FINAL_ZIVE_SECURITY_TYPE_PROMPT).content
pattern = r'<output>(.*?)</output>'
match = re.search(pattern, output, re.DOTALL)
final_data = match.group(1).strip()
return final_data
def process_financial_records(investments_df):
"""
Process and insert users from an Excel file into MongoDB.
Args:
file_path (str): Path to the Excel file.
"""
entity_id = os.getenv('ENTITY_ID')
fund_id = os.getenv('FUND_ID')
# print(f"Header:\n {partners_df.head()}")
investments = []
# current_company = None
# portfoltio_company_id = None
# investment_number = 0
for investment, details in investments_df.items():
print(f"Adding company {investment}")
payload = {
"fundId": fund_id,
"companyName": investment
}
print(f"\nInvestment: {investment}")
response = add_portfolio_company(payload)
if 'error' in response:
print(f"Failed to fetch onboarding status: Error: {response['error']}")
print(f"Status Code: {response['status_code']}")
continue
portfoltio_company_id = response["data"]["data"]["_id"]
for security in details['securities']:
print(f" Security Type: {security['type']}")
print(f" Investment Date: {security['investment_date']}")
print(f" Investment Number: {security['investment_number']}")
investment = {
"fundId": fund_id,
"portfolioId": portfoltio_company_id,
"investmentNumber": security['investment_number'],
"investmentDate": security['investment_date'],
"securityType": get_security_type(security['type']),
"assetName": security['type'],
"asset": security['type']
}
print(investment)
response = add_portfolio_company_investment(investment)
if 'error' in response:
print(response)
print(f"Failed to fetch onboarding status: Error: {response['error']}")
print(f"Status Code: {response['status_code']}")
continue # Process next company investment record...
investments.append(investment)
# Iterate through the DataFrame and prepare partner records
# for index, row in financials_df.iterrows():
#
# if current_company != row['Investment']:
# current_company = row['Investment']
# investment_number = 0
#
# # make the call to add portfolio company
# print(f"Adding company {current_company}")
# payload = {
# "fundId": fund_id,
# "companyName": current_company
# }
# response = add_portfolio_company(payload)
# if 'error' in response:
# print(f"Failed to fetch onboarding status: Error: {response['error']}")
# print(f"Status Code: {response['status_code']}")
# break #Process next company record...
# portfoltio_company_id = response["data"]["data"]["_id"]
#
#
# investment_number += 1
# asset_type = row['Asset type'] if pd.notna(row['Asset type']) else None
#
# investment = {
# "fundId": fund_id,
# "portfolioId": portfoltio_company_id,
# "investmentNumber": investment_number,
# "investmentDate": row['Investment date'] if pd.notna(row['Investment date']) else None,
# "securityType": get_security_type(asset_type),
# "assetName": row['Asset'] if pd.notna(row['Asset']) else None,
# "asset": asset_type
# }
#
# response = add_portfolio_company_investment(investment)
# if 'error' in response:
# print(f"Failed to fetch onboarding status: Error: {response['error']}")
# print(f"Status Code: {response['status_code']}")
# break # Process next company investment record...
# investments.append(investment)
# print(f"Adding portforlio investment {index} {current_company}:{portfoltio_company_id}:\n {investment}")
# Test with smaller set of records
# if index == 1:
# break
return investments
def process_financials(file_path):
# Process partner data
step_number = 4
step_number -= 1
success_message = ""
error_message = ""
status = "IN-PROGRESS"
bucket_name = os.getenv('S3_UPLOAD_BUCKET_NAME')
print(f"Processing file: {file_path}")
response = update_onboarding_status(step_number, status, error_message, success_message)
if 'error' in response:
print(f"Failed to fetch onboarding status: Error: {response['error']}")
print(f"Status Code: {response['status_code']}")
return False
partner_excel_file = read_file_from_s3(bucket_name, file_path)
sheet_name = 'Roll Forward'
financial_df = xlsx_to_df(partner_excel_file, sheet_name)
# print(financial_df.head())
financials = process_financial_records(financial_df)
if financials:
item_count = len(financials)
success_message = f"Onboarded {item_count} portfolio financial records."
status = "COMPLETE"
else:
error_message = "No transactions were onboaerded."
status = "FAILED"
response = update_onboarding_status(step_number, status, error_message, success_message)
if 'error' in response:
print(f"Failed to fetch onboarding status: Error: {response['error']}")
print(f"Status Code: {response['status_code']}")
return False
# print(f"Investments list:\n {financials}")
# print(json.dumps(partners_dict, indent=2, sort_keys=True, cls=CustomJSONEncoder))
return True

279
task/onboardFunds.py Normal file
View File

@ -0,0 +1,279 @@
from PIL import Image
import boto3
import fitz
import re
import io
import os
import json
from llm import get_llm_client
import tiktoken
import base64
import time
from s3Ops import (
read_file_from_s3
)
def pdf_to_image(pdf_key, input_bucket_name, output_bucket_name):
s3 = boto3.client('s3')
print(f"Reading file from {input_bucket_name}, with key {pdf_key}")
# Load the PDF file into memory
pdf_object = s3.get_object(Bucket=input_bucket_name, Key=pdf_key)
pdf_content = pdf_object['Body'].read()
# Extract file name from pdf_key
file_name = pdf_key.split('/')[-1]
output_file_name = file_name.rsplit('.', 1)[0] # Remove file extension
image_list = []
# Open the PDF
pdf_document = fitz.open(stream=pdf_content, filetype="pdf")
for page_num in range(len(pdf_document)):
page = pdf_document.load_page(page_num)
scale = 1.8
pix = page.get_pixmap(matrix=fitz.Matrix(scale, scale))
# Convert pixmap to PNG bytes directly
img_bytes = pix.tobytes("jpg")
image_key = f"{output_file_name}/{output_file_name}_{page_num + 1}.jpg"
image_list.append(image_key)
# Upload the image to S3
s3.put_object(Bucket=output_bucket_name, Key=image_key, Body=img_bytes, ContentType='image/jpg')
return image_list
def get_images(image_key, bucket):
s3 = boto3.client('s3')
# Get the object from S3
response = s3.get_object(Bucket=bucket, Key=image_key)
# Read the content of the file
image_data = response['Body'].read()
return image_data
# for use with OCR/last pages
def create_message(extracted_texts, prompt, json_template, last_images64):
message = {
"role": "user",
"content": [
{
"type": "text",
"text": f"Prompt:\n{prompt}\n\n"
f"JSON Template:\n```json\n{json_template}```\n\n"
f"Text to analyze:\n"
}
]
}
# Add extracted texts
for index, text in enumerate(extracted_texts):
message["content"].append({
"type": "text",
"text": f"Page {index + 1} limited partnership agreement document:\n{text}\n"
})
# Add a separator before images
message["content"].append({
"type": "text",
"text": "\nLast ten corresponding images to analyze:\n"
})
# Add the last few images to the message
for i, image_data in enumerate(last_images64):
# Add the image
message["content"].append({
"type": "image",
"source": {
"type": "base64",
"media_type": "image/jpeg",
"data": image_data
}
})
# Add a text description for each image
message["content"].append({
"type": "text",
"text": f"Image {i + 1} (corresponds to Page {len(extracted_texts) - len(last_images64) + i + 1})\n"
})
return message
def ask_llm(message):
# increase timeout
# (boto3/Config removed — the Gemini client manages its own timeouts)
# initialize unified Gemini LLM client (.invoke_model() drop-in; see task/llm.py)
bedrock = get_llm_client()
# construct the request body for bedrock API
body = json.dumps({
"max_tokens": 16384,
"system": "You are a financial advisor specializing in extracting and evaluating key information from financial documents. We are discussing terms within a limited partnership agreement document.",
"messages": [
message,
{"role": "assistant", "content": '```json'} # Prefill here
],
"anthropic_version": "bedrock-2023-05-31"
})
# invoke the model
response = bedrock.invoke_model(body=body, modelId="anthropic.claude-3-5-sonnet-20240620-v1:0")
# parse the response
response_body = json.loads(response.get("body").read())
# return the OCR result
return response_body['content'][0]['text']
def detailed_token_count(message, model="cl100k_base"):
enc = tiktoken.get_encoding(model)
total_tokens = 0
breakdown = {}
# Count tokens for the role
role_tokens = len(enc.encode(message["role"]))
total_tokens += role_tokens
breakdown["role"] = role_tokens
# Count tokens for each content item
for i, item in enumerate(message["content"]):
if item["type"] == "text":
text_tokens = len(enc.encode(item["text"]))
total_tokens += text_tokens
breakdown[f"content_{i}"] = text_tokens
# Add tokens for the "type" field itself
type_tokens = len(enc.encode(item["type"]))
total_tokens += type_tokens
breakdown[f"type_{i}"] = type_tokens
return total_tokens, breakdown
def clean_json(text):
# Find the start and end of the JSON object
start = output.find('{')
end = output.rfind('}') + 1
# Extract the JSON string
json_str = output[start:end]
# Parse the JSON string to ensure it's valid
try:
json.loads(json_str)
except json.JSONDecodeError:
return None # Return None if the JSON is invalid
return json_str
def add_backtack(text):
# Find the position of the first '{'
start_pos = text.find('{')
# If there's no '{', return the original text
if start_pos == -1:
return text
# Find the start of the line containing '{'
line_start = text.rfind('\n', 0, start_pos)
if line_start == -1:
line_start = 0
else:
line_start += 1 # Move past the newline character
# Insert '```json\n' before the line containing '{'
return text[:line_start] + '```json\n' + text[line_start:]
def onboardFunds(lpa_file_path):
input_bucket = os.getenv('S3_UPLOAD_BUCKET_NAME')
output_bucket = input_bucket
image_keys = pdf_to_image(lpa_file_path, input_bucket, output_bucket)
images = []
for image_key in image_keys:
image = get_images(image_key, output_bucket)
images.append(image)
print(image_keys[-6:-1])
os.environ['TESSDATA_PREFIX'] = 'RTC:pdf processing/eng.traineddata'
extracted_texts2 = []
start_time = time.time()
for i, image in enumerate(images, 1):
# Open the image using PIL
img = Image.open(io.BytesIO(image)).convert('L')
# Append the extracted text to the list
extracted_texts2.append(text)
# Print status update every 10 images
if i % 10 == 0:
elapsed_time = time.time() - start_time
print(f"Processed {i} images out of {len(images)} in {elapsed_time:.2f} seconds")
# Calculate total time
total_time = time.time() - start_time
# Print final status after processing all images
print(f"Finished processing all {len(images)} images in {total_time:.2f} seconds")
print(f"Average time per image: {total_time/len(images):.2f} seconds")
last_images64 = []
for image in images[-10:]:
image = base64.b64encode(image).decode("utf-8")
last_images64.append(image)
print(len(extracted_texts2))
print(extracted_texts2[-2])
message = create_message(extracted_texts2, prompt2, json_template, last_images64)
#print(f"\nFinal message structure:")
#print(f"Number of content items: {len(message['content'])}")
#for i, item in enumerate(message['content']):
# print(f"Content {i} length: {len(item['text'])}")
# Use the detailed token count function
total_count, breakdown = detailed_token_count(message)
print(f"\nTotal token count: {total_count}")
#print("Token breakdown:")
#for key, value in breakdown.items():
# print(f" {key}: {value}")
outputs = ask_llm(message)
print(outputs)
if __name__ == "__main__":
from dotenv import load_dotenv
load_dotenv()
onboardingId = "66ea163564e2f97a059160ef"
os.environ["ONBOARDING_ID"] = onboardingId
from initOnboarding import initialize_onboarding
initialize_onboarding()
found_files = {'lpa': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/0-Please_DocuSign_CerraCap_II_LP_Limited_Partn.pdf', 'partner': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/0-1-cerracap-ii-lp_2024-07_09_short_partner.xlsx', 'financials': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/2-3-cerracap-ii-lp_2024-08-26_financials.xlsx', 'bankTransactions': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/4-CerraCap_II__LP_bank_transactions_2016-01-01-2024-07-03.xlsx', 'journals': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/5-cerracap-ii-lp_2024-07-09_journals-export.xlsx', 'fund_performance': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/6-cerracap-ii-lp_2024-07-09_fund-performance-report.xlsx'}
lpa_file_path = found_files["lpa"]
onboardFunds(lpa_file_path)

132
task/onboardFundsDummy.py Normal file
View File

@ -0,0 +1,132 @@
import io
import os
import json
import base64
import time
from s3Ops import read_file_from_s3
import pandas as pd
import re
from backendAPIs import (
update_fund,
update_onboarding_status
)
def clean_currency(x):
try:
if isinstance(x, str):
return float(re.sub(r'[\$,]', '', x))
return float(x)
except:
return 0 # or np.nan if you prefer
def get_partner_summary(df):
# Calculate values
total_commitment = df['Commitment'].apply(clean_currency).sum()
earliest_date = pd.to_datetime(df['Issue date']).min()
general_partner = df[df['Class'] == 'General Partner']['Partner'].iloc[0]
# Create dictionary with results
summary_dict = {
'total_commitments': total_commitment,
'earliest_issue_date': earliest_date.strftime('%m/%d/%Y'),
'general_partner': general_partner
}
return summary_dict
def get_partners_from_excel(xlsx_file):
"""
Read Excel file containing pa information.
Remove headers and return clean df
Args:
file_path (str): Path to the Excel file.
sheet_name (str): Excel sheet name with partner list
Returns:
pd.DataFrame: DataFrame containing partner information.
"""
sheet_name = 'Partners'
df = pd.read_excel(xlsx_file, sheet_name = sheet_name)
df.columns = df.iloc[2]
df = df.drop(df.index[:3])
df = df.reset_index(drop=True)
fields_dict = get_partner_summary(df)
return fields_dict
def onboardFunds(lpa_file_path):
# input_bucket = os.getenv('S3_UPLOAD_BUCKET_NAME')
step_number = 2
step_number -= 1
success_message = ""
error_message = ""
status = "IN-PROGRESS"
print("IN-PROGREE")
response = update_onboarding_status(step_number, status, error_message, success_message)
bucket_name = os.getenv('S3_UPLOAD_BUCKET_NAME')
print(f"Processing file: {lpa_file_path}")
response = update_onboarding_status(step_number, status, error_message, success_message)
if 'error' in response:
print(f"Failed to fetch onboarding status: Error: {response['error']}")
print(f"Status Code: {response['status_code']}")
return False
partner_excel_file = read_file_from_s3(bucket_name,lpa_file_path)
partners_df = get_partners_from_excel(partner_excel_file)
print(partners_df)
fund_data = {
"fundSize": partners_df["total_commitments"],
"dateFormed": partners_df["earliest_issue_date"],
"fundDuration": "10",
"gpName": partners_df["general_partner"],
"investmentperiodmanagementFee": "2%",
"mgmtCoName": "Test Mgt Co",
"postInvestmentPeriodmanagementFee": "4%"
}
response = update_fund(fund_data)
print(f"Response: \n {response}")
fund_id = response["data"]["data"]["_id"]
print(f"Setting fund id in the environment {fund_id}")
os.environ["FUND_ID"] = fund_id
if fund_id:
success_message = f"Onboarded fund {fund_id} journal records."
status = "COMPLETE"
else:
error_message = "No journals were onboaerded."
status = "FAILED"
response = update_onboarding_status(step_number, status, error_message, success_message)
if 'error' in response:
print(f"Failed to fetch onboarding status: Error: {response['error']}")
print(f"Status Code: {response['status_code']}")
return False
return fund_id
if __name__ == "__main__":
from dotenv import load_dotenv
load_dotenv()
onboardingId = "66ea163564e2f97a059160ef"
os.environ["ONBOARDING_ID"] = onboardingId
from initOnboarding import initialize_onboarding
initialize_onboarding()
found_files = {'lpa': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/0-Please_DocuSign_CerraCap_II_LP_Limited_Partn.pdf', 'partner': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/0-1-cerracap-ii-lp_2024-07_09_short_partner.xlsx', 'financials': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/2-3-cerracap-ii-lp_2024-08-26_financials.xlsx', 'bankTransactions': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/4-CerraCap_II__LP_bank_transactions_2016-01-01-2024-07-03.xlsx', 'journals': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/5-cerracap-ii-lp_2024-07-09_journals-export.xlsx', 'fund_performance': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/6-cerracap-ii-lp_2024-07-09_fund-performance-report.xlsx'}
lpa_file_path = found_files["lpa"]
onboardFunds(lpa_file_path)

349
task/onboardJournals.py Normal file
View File

@ -0,0 +1,349 @@
import pandas as pd
import json
import os
from utils.prompts import ACCOUNT_TYPE_PROMPT
from utils.account_type import account_type_data
import re
from s3Ops import read_file_from_s3
from backendAPIs import (
update_onboarding_status,
add_journal,
add_journal_ledger
)
# LLM via the unified Gemini-only backend (see task/llm.py).
from llm import get_chat_model
model_parameter = {"temperature": 0, "max_tokens": 16384}
llm_bedrock = get_chat_model(model_kwargs=model_parameter)
def xlsx_to_df(xlsx_file, sheet_name):
df = pd.read_excel(xlsx_file, sheet_name=sheet_name, header=2, skiprows=2)
return df
# TODO: add the remaining asset type mapping
def get_security_type(asset_type):
asset_to_security_type = {
"Common stock": "COMMON",
"Preferred stock": "PREFERRED",
"Warrants": "WARRANTS",
"Convertible promissory note/SAFEs": "POST MONEY SAFE"
}
for asset, security_type in asset_to_security_type.items():
if asset_type.lower() in asset.lower(): # Case-insensitive match
return security_type
return ""
def get_account_type(account_type):
print(account_type.split("-")[0])
print(account_type.split("-")[1].lower().strip())
if account_type.split("-")[0].strip() == "1000" and account_type.split("-")[1].lower().strip() == "bank":
return "Bank"
result = account_type.split("-", 1)
# Strip whitespace from both parts
account_lookup_string = [part.strip() for part in result][1]
print(account_lookup_string)
if len(account_type_data.get(account_lookup_string.strip(), "")) > 0:
print("from dict")
return account_type_data.get(account_lookup_string.strip())
else:
print("llm call")
FINAL_ZIVE_ACCOUNT_TYPE_PROMPT = ACCOUNT_TYPE_PROMPT.format(
account_type=account_type)
output = llm_bedrock.invoke(FINAL_ZIVE_ACCOUNT_TYPE_PROMPT).content
pattern = r'<output>(.*?)</output>'
match = re.search(pattern, output, re.DOTALL)
final_data = match.group(1).strip()
return final_data
def process_journal_records(df):
"""
Process and insert users from an Excel file into MongoDB.
Args:
file_path (str): Path to the Excel file.
"""
entity_id = os.getenv('ENTITY_ID')
fund_id = os.getenv('FUND_ID')
# Filter out rows with invalid Journal IDs
# df = df[~df['Journal ID'].isin(['Journal ID'])]
# print(df)
journals = []
current_journal = None
journal_id = None
journal_number = 0
jcredit = 0
jdebit = 0
jdate = None
jeventType = None
jdescription = None
jid = None
jls = []
# Iterate through the DataFrame and prepare partner records
for index, row in df.iterrows():
if pd.notna(row['Journal ID']):
if current_journal is not None and current_journal != row['Journal ID']:
current_journal = row['Journal ID']
# journal_number = 0
# make the call to add portfolio company
# print(f"Adding company {current_journal}")
# Handle NaN values in description field
if pd.isna(jdescription) or jdescription is None or str(jdescription).lower() == 'nan':
jdescription = "Migration"
payload = {
"dateOfJournalEntry": jdate,
"eventType": jeventType,
"entryDescription": jdescription,
"debitAmount": jdebit,
"creditAmount": jcredit,
}
# print(payload)
print(payload)
try:
response = add_journal(payload)
print(response)
# Check if the response was successful
if 'error' in response:
print(
f"ERROR: Failed to add journal entry: {response.get('error', 'Unknown error')}")
print(
f"Status Code: {response.get('status_code', 'N/A')}")
# Only skip THIS journal's ledger entries, not the entire process
# The next journal (with different Journal ID) will still be processed
print(
f"Skipping journal {current_journal}'s ledger entries but continuing with next journal...")
else:
# Only process ledger entries if journal was successfully created
if response and "data" in response and "data" in response["data"] and "_id" in response["data"]["data"]:
journal_id = response["data"]["data"]["_id"]
journals.append(payload)
# Process ledger entries for this journal
for roww in jls:
print(roww['accountType'])
try:
add_journal_ledger({
"accountType": get_account_type(roww['accountType']),
"portfolioCompany": roww['portfolioCompany'] if pd.notna(roww['portfolioCompany']) else '',
"partner": roww['partner'] if pd.notna(roww['partner']) else '',
"investmentInfo": roww['investmentInfo'] if pd.notna(roww['investmentInfo']) else '',
"debitAmount": roww['debitAmount'],
"creditAmount": roww['creditAmount'],
"shares": roww.get('shares', 0),
"journalId": journal_id
})
except Exception as ledger_error:
print(
f"ERROR: Failed to add journal ledger entry: {ledger_error}")
print("Continuing with next ledger entry...")
continue
else:
print(
f"WARNING: Journal created but no ID returned. Response: {response}")
except Exception as e:
print(f"ERROR: Exception while adding journal: {e}")
print(
"Skipping this journal's ledger entries but continuing with next journal...")
# Reset accumulators for next journal and add current row as first entry of next journal
jls = []
jcredit = 0
jdebit = 0
jls.append({
"accountType": row['Account'],
"portfolioCompany": row['Issuer'],
"investmentInfo": row['Asset'],
"partner": row['Partner'],
"debitAmount": row['Debit'],
"creditAmount": row['Credit'],
"shares": row['Shares'] if pd.notna(row['Shares']) else 0
})
jcredit += row['Credit']
jdebit += row['Debit']
jdate = row['Journal date']
jeventType = row['Event type']
jdescription = row['Description']
# print(response["data"])
# Removed the old error handling as it's now handled above
# journal_id = response["data"]["data"]["_id"]
# print(f"Adding journal:\n {payload}")
# journal_number += 1
else:
current_journal = row['Journal ID']
jcredit += row['Credit']
jdebit += row['Debit']
jdate = row['Journal date']
jeventType = row['Event type']
jdescription = row['Description']
jls.append({
"accountType": row['Account'],
"portfolioCompany": row['Issuer'],
"investmentInfo": row['Asset'],
"partner": row['Partner'],
"debitAmount": row['Debit'],
"creditAmount": row['Credit'],
"shares": row['Shares'] if pd.notna(row['Shares']) else 0
})
# response = add_journal_ledger(journal_ledger)
# print(f"Adding joural ledger {index} {current_journal}:{journal_id}:\n {journal_ledger}\n\n")
# if 'error' in response:
# print(f"Failed to fetch onboarding status: Error: {response['error']}")
# print(f"Status Code: {response['status_code']}")
# break # Process next company investment record...
# journals.append(journal_ledger)
# # Test with smaller set of records
# if index == 1:
# break
# Process the last journal entry after the loop
if len(jls) > 0:
# Handle NaN values in description field
if pd.isna(jdescription) or jdescription is None or str(jdescription).lower() == 'nan':
jdescription = ""
payload = {
"dateOfJournalEntry": jdate,
"eventType": jeventType,
"entryDescription": jdescription,
"debitAmount": jdebit,
"creditAmount": jcredit,
}
print(payload)
try:
response = add_journal(payload)
print(response)
# Check if the response was successful
if 'error' in response:
print(
f"ERROR: Failed to add last journal entry: {response.get('error', 'Unknown error')}")
print(f"Status Code: {response.get('status_code', 'N/A')}")
else:
# Only process ledger entries if journal was successfully created
if response and "data" in response and "data" in response["data"] and "_id" in response["data"]["data"]:
journal_id = response["data"]["data"]["_id"]
journals.append(payload)
# Process ledger entries for this journal
for roww in jls:
print(roww['accountType'])
try:
add_journal_ledger({
"accountType": get_account_type(roww['accountType']),
"portfolioCompany": roww['portfolioCompany'] if pd.notna(roww['portfolioCompany']) else '',
"partner": roww['partner'] if pd.notna(roww['partner']) else '',
"investmentInfo": roww['investmentInfo'] if pd.notna(roww['investmentInfo']) else '',
"debitAmount": roww['debitAmount'],
"creditAmount": roww['creditAmount'],
"shares": roww.get('shares', 0),
"journalId": journal_id
})
except Exception as ledger_error:
print(
f"ERROR: Failed to add journal ledger entry: {ledger_error}")
print("Continuing with next ledger entry...")
continue
else:
print(
f"WARNING: Last journal created but no ID returned. Response: {response}")
except Exception as e:
print(f"ERROR: Exception while adding last journal: {e}")
return journals
def process_journals(file_path):
# Process partner data
step_number = 6
step_number -= 1
success_message = ""
error_message = ""
status = "IN-PROGRESS"
bucket_name = os.getenv('S3_UPLOAD_BUCKET_NAME')
print(f"Processing file: {file_path}")
response = update_onboarding_status(
step_number, status, error_message, success_message)
if 'error' in response:
print(f"Failed to fetch onboarding status: Error: {response['error']}")
print(f"Status Code: {response['status_code']}")
return False
journal_excel_file = read_file_from_s3(bucket_name, file_path)
sheet_name = 'Posted journals'
financial_df = xlsx_to_df(journal_excel_file, sheet_name)
print("financial_df")
print(financial_df)
print(financial_df.columns)
journals = process_journal_records(financial_df)
if journals:
item_count = len(journals)
success_message = f"Onboarded {item_count} journal records."
status = "COMPLETE"
else:
error_message = "No journals were onboaerded."
status = "FAILED"
response = update_onboarding_status(
step_number, status, error_message, success_message)
if 'error' in response:
print(f"Failed to fetch onboarding status: Error: {response['error']}")
print(f"Status Code: {response['status_code']}")
return False
print(f"Journals list:\n {journals}")
# print(json.dumps(partners_dict, indent=2, sort_keys=True, cls=CustomJSONEncoder))
return True
if __name__ == "__main__":
from dotenv import load_dotenv
load_dotenv()
onboardingId = "66ea163564e2f97a059160ef"
os.environ["ONBOARDING_ID"] = onboardingId
os.environ["FUND_ID"] = "66c5e6d89ecbf552a05b84fc"
from initOnboarding import initialize_onboarding
initialize_onboarding()
found_files = {'lpa': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/0-Please_DocuSign_CerraCap_II_LP_Limited_Partn.pdf', 'partner': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/0-1-cerracap-ii-lp_2024-07_09_short_partner.xlsx', 'financials': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/2-3-cerracap-ii-lp_2024-08-26_financials.xlsx',
'bankTransactions': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/4-CerraCap_II__LP_bank_transactions_2016-01-01-2024-07-03.xlsx', 'journals': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/5-cerracap-ii-lp_2024-07-09_journals-export.xlsx', 'fund_performance': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/6-cerracap-ii-lp_2024-07-09_fund-performance-report.xlsx'}
financials_excel_file_path = found_files["journals"]
process_journals(financials_excel_file_path)

240
task/onboardPartners.py Normal file
View File

@ -0,0 +1,240 @@
import pandas as pd
import json
import os
import re
# from bson import ObjectId
# from pymongo.results import UpdateResult
from s3Ops import read_file_from_s3
from backendAPIs import (
update_onboarding_status,
add_role
)
# class CustomJSONEncoder(json.JSONEncoder):
# def default(self, obj):
# if isinstance(obj, ObjectId):
# return str(obj) # Convert ObjectId to string for JSON serialization
# elif isinstance(obj, UpdateResult):
# # Convert UpdateResult to a serializable dictionary
# return {
# "matched_count": obj.matched_count,
# "modified_count": obj.modified_count,
# "upserted_id": str(obj.upserted_id) if obj.upserted_id else None,
# "acknowledged": obj.acknowledged
# }
# # For other non-serializable objects, use the default behavior
# return super().default(obj)
def get_partners_from_excel(xlsx_file):
"""
Read Excel file containing pa information.
Remove headers and return clean df
Args:
file_path (str): Path to the Excel file.
sheet_name (str): Excel sheet name with partner list
Returns:
pd.DataFrame: DataFrame containing partner information.
"""
sheet_name = 'Partners'
df = pd.read_excel(xlsx_file, sheet_name = sheet_name)
df.columns = df.iloc[2]
df = df.drop(df.index[:3])
df = df.reset_index(drop=True)
return df
def pretty_print_files(files_dict):
"""
Pretty print the dictionary containing file names.
Args:
files_dict (dict): Dictionary where the keys are the file categories, and the values are the file names.
"""
file_message = ""
for key, value in files_dict.items():
file_message += f"- {key.replace('_', ' ').capitalize()}: {value} \n"
return file_message
def process_partners(partners_df):
"""
Process and insert users from an Excel file into MongoDB.
Args:
file_path (str): Path to the Excel file.
"""
entity_id = os.getenv('ENTITY_ID')
# print(f"Header:\n {partners_df.head()}")
users = []
roles = []
# Iterate through the DataFrame and prepare partner records
for index, row in partners_df.iterrows():
# Skip rows without a valid Partner name
if pd.isna(row['Partner']) in ['Partner']:
print(f"Skipping row {index}")
continue
partner_dict = {
"firstName": row['Primary Contact Name'].split(' ')[0] if pd.notna(row['Primary Contact Name']) else '',
"lastName": ' '.join(row['Primary Contact Name'].split(' ')[1:]) if pd.notna(row['Primary Contact Name']) else '',
"email": row['Email'] if pd.notna(row['Email']) else ''
}
users.append(partner_dict)
print(f"Adding user {partner_dict}")
# Insert user record into user collection
# user_id = add_user(partner_dict)
# print(json.dumps(insert_result, indent=2))
# Adding role automatiocally adds users
role, role_id = process_roles(row)
roles.append(role)
print(role_id)
return users
def remove_all_special_chars(text):
return re.sub(r'[^a-zA-Z0-9\s]', '', text)
def process_roles(row):
"""
Process and insert users from an Excel file into MongoDB.
Args:
file_path (str): Path to the Excel file.
"""
entity_id = os.getenv('ENTITY_ID')
accessType, role = "", ""
if pd.notna(row['Class']): # Check if the Class value is not NaN
if row['Class'] == 'Limited Partner' or row['Class'] == 'Member':
accessType = 'USER'
role = "LIMITED PARTNER"
elif row['Class'] == 'General Partner':
accessType = 'ADMIN'
role = "GENERAL PARTNER"
entity_name = row['Partner'] if any(indicator.lower() in row['Partner'].lower() for indicator in ["corp", "trust", "international", "llc", "ltd", "council", "inc", "llp", "sp", "lp", "capital", "fund","foundation","and","&","l.p."]) or any(char.isdigit() for char in row['Partner']) or re.search(r'\d{1,2}/\d{1,2}/\d{2,4}', row['Partner']) else ''
if len(entity_name) > 0:
first_name = row['Primary Contact Name'].split(' ')[0] if pd.notna(row['Primary Contact Name']) else ''
last_name = ' '.join(row['Primary Contact Name'].split(' ')[1:]) if pd.notna(row['Primary Contact Name']) else ''
else:
first_name = row['Partner'].split(' ')[0] if pd.notna(row['Partner']) else ''
last_name = ' '.join(row['Partner'].split(' ')[1:]) if pd.notna(row['Partner']) else ''
role_dict = {
# "userId": {
# "$oid": user_id
# },
# "entityId": {
# "$oid": entity_id
# },
"entityId": entity_id,
"entityName":entity_name,
"accessType": accessType,
"role": role,
"firstName": first_name,
"lastName": last_name,
# "status": "NOT INVITED",
# "isInvitationSent": False,
# "dob": None,
"phoneNumber": row['Phone'].split(' ')[0] if pd.notna(row['Phone']) else '',
"taxID1": row['Tax ID'].split(' ')[0] if pd.notna(row['Tax ID']) else '',
"taxID2": row['Tax ID Type'].split(' ')[0] if pd.notna(row['Tax ID Type']) else '',
"street": remove_all_special_chars(row['Street Address'].split(' ')[0]) if pd.notna(row['Street Address']) else '',
"address": remove_all_special_chars(" ".join(row['Street Address'].split(' ')[1:])) if pd.notna(row['Street Address']) else '',
"country":row['Country'].split(' ')[0] if pd.notna(row['Country']) else '',
"city": row['City'].split(' ')[0] if pd.notna(row['City']) else '',
"state": row['State'].split(' ')[0] if pd.notna(row['State']) else '',
"zipcode": row['ZIP'].split(' ')[0] if pd.notna(row['ZIP']) else '',
"commitedAmount": row['Commitment'],
"calledCapital": row['Called Capital'],
"dateOfCommitment": row['Issue date'],
"partnerExactName": row['Partner'],
"email":row['Email'] if pd.notna(row['Email']) else ''
# "gender": "",
# "ethnicity": "",
# "photoURL": None,
# "menuOpen": False
}
print("entityName")
print(entity_name)
print(role_dict)
# Save role into database
role_id = add_role(role_dict)
print(f"Upserted Role ID: {role_id}")
return role_dict, role_id
def process_all_partners(partners_file_path):
# Process partner data
step_number = 3
step_number -= 1
success_message = ""
error_message = ""
status = "IN-PROGRESS"
bucket_name = os.getenv('S3_UPLOAD_BUCKET_NAME')
print(f"Processing file: {partners_file_path}")
response = update_onboarding_status(step_number, status, error_message, success_message)
if 'error' in response:
print(f"Failed to fetch onboarding status: Error: {response['error']}")
print(f"Status Code: {response['status_code']}")
return False
partner_excel_file = read_file_from_s3(bucket_name, partners_file_path)
partners_df = get_partners_from_excel(partner_excel_file)
print(partners_df.head())
partners_dict = process_partners(partners_df)
if partners_dict:
item_count = len(partners_dict)
success_message = f"Onboarded {item_count} GPs&LPs."
status = "COMPLETE"
else:
error_message = "No users were onboaerded."
status = "FAILED"
print("*"*30)
response = update_onboarding_status(step_number, status, error_message, success_message)
if 'error' in response:
print(f"Failed to fetch onboarding status: Error: {response['error']}")
print(f"Status Code: {response['status_code']}")
return False
# print("Partner list:")
# print(json.dumps(partners_dict, indent=2, sort_keys=True, cls=CustomJSONEncoder))
return True
if __name__ == "__main__":
from dotenv import load_dotenv
load_dotenv()
onboardingId = "66ea163564e2f97a059160ef"
os.environ["ONBOARDING_ID"] = onboardingId
from initOnboarding import initialize_onboarding
initialize_onboarding()
found_files = {'lpa': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/0-Please_DocuSign_CerraCap_II_LP_Limited_Partn.pdf', 'partner': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/0-1-cerracap-ii-lp_2024-07_09_short_partner.xlsx', 'financials': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/2-3-cerracap-ii-lp_2024-08-26_financials.xlsx', 'bankTransactions': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/4-CerraCap_II__LP_bank_transactions_2016-01-01-2024-07-03.xlsx', 'journals': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/5-cerracap-ii-lp_2024-07-09_journals-export.xlsx', 'fund_performance': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/6-cerracap-ii-lp_2024-07-09_fund-performance-report.xlsx'}
partner_excel_file_path = found_files["partner"]
process_all_partners(partner_excel_file_path)

125
task/onboardPerformance.py Normal file
View File

@ -0,0 +1,125 @@
import pandas as pd
import json
import os
from s3Ops import read_file_from_s3
from backendAPIs import (
update_onboarding_status,
add_performance_record
)
def xlsx_to_df(xlsx_file, sheet_name):
df = pd.read_excel(xlsx_file, sheet_name = sheet_name, header=2,skiprows=2)
return df
def process_performance_records(df):
"""
Process and insert users from an Excel file into MongoDB.
Args:
file_path (str): Path to the Excel file.
"""
entity_id = os.getenv('ENTITY_ID')
fund_id = os.getenv('FUND_ID')
role_id = os.getenv('ROLE_ID')
performance_records = []
# Iterate through the DataFrame and prepare partner records
for index, row in df.iterrows():
print(row.to_dict())
if row['Type'].strip().lower() == "contribution":
continue
tdate = row['Date']
performance_record = {
"fundId": fund_id,
"entityId": entity_id,
"roleId": role_id,
"receivedDate": tdate.strftime('%m/%d/%Y'),
"amount": row['Value'] if row['Value'] >= 0 else row['Value']*-1,
"gainLoss": "GAIN" if row['Value'] >= 0 else "LOSS",
"type": row['Type'],
"partnerExactName": row["Partner"]
}
response = add_performance_record(performance_record)
print(f"Adding joural ledger {index}\n {performance_record}\n\n")
if 'error' in response:
print(response)
print(f"Failed to fetch onboarding status: Error: {response['error']}")
print(f"Status Code: {response['status_code']}")
# break # Process next company investment record...
continue
performance_records.append(performance_record)
print(f"Performance record: \n{performance_record}")
# Test with smaller set of records
# if index == 1:
# break
return performance_records
def process_fund_performance(file_path):
# Process partner data
step_number = 7
step_number -= 1
success_message = ""
error_message = ""
status = "IN-PROGRESS"
bucket_name = os.getenv('S3_UPLOAD_BUCKET_NAME')
print(f"Processing file: {file_path}")
response = update_onboarding_status(step_number, status, error_message, success_message)
if 'error' in response:
print(f"Failed to fetch onboarding status: Error: {response['error']}")
print(f"Status Code: {response['status_code']}")
return False
performance_excel_file = read_file_from_s3(bucket_name, file_path)
sheet_name = 'Partner Capital Activity Detail'
performance_df = xlsx_to_df(performance_excel_file, sheet_name)
print(performance_df)
perf_records = process_performance_records(performance_df)
if perf_records:
item_count = len(perf_records)
success_message = f"Onboarded {item_count} fund performance records."
status = "COMPLETE"
else:
error_message = "No transactions were onboaerded."
status = "FAILED"
response = update_onboarding_status(step_number, status, error_message, success_message)
if 'error' in response:
print(f"Failed to fetch onboarding status: Error: {response['error']}")
print(f"Status Code: {response['status_code']}")
return False
# print(f"Journals list:\n {perf_records}")
return True
if __name__ == "__main__":
from dotenv import load_dotenv
load_dotenv()
onboardingId = "66ea163564e2f97a059160ef"
os.environ["ONBOARDING_ID"] = onboardingId
os.environ["FUND_ID"] = "66c5e6d89ecbf552a05b84fc"
from initOnboarding import initialize_onboarding
initialize_onboarding()
found_files = {'lpa': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/0-Please_DocuSign_CerraCap_II_LP_Limited_Partn.pdf', 'partner': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/0-1-cerracap-ii-lp_2024-07_09_short_partner.xlsx', 'financials': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/2-3-cerracap-ii-lp_2024-08-26_financials.xlsx', 'bankTransactions': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/4-CerraCap_II__LP_bank_transactions_2016-01-01-2024-07-03.xlsx', 'journals': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/5-cerracap-ii-lp_2024-07-09_journals-export.xlsx', 'fund_performance': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/6-cerracap-ii-lp_2024-07-09_fund-performance-report.xlsx'}
financials_excel_file_path = found_files["fund_performance"]
process_fund_performance(financials_excel_file_path)

177
task/prompt.py Normal file
View File

@ -0,0 +1,177 @@
prompt2 = """
Analyze the following limited partnership agreement document and update the provided JSON object with relevant information. Apply your expertise in financial document analysis and understanding of limited partnership agreements throughout this task.
Key Instructions:
1. Carefully read and interpret the entire document, including any amendments.
2. Extract all relevant information matching the JSON structure fields.
3. Update JSON values with newly extracted information, interpreting complex terms and conditions.
4. Apply critical thinking to deduce JSON values when not explicitly stated.
5. Only update values in the JSON, keeping all original fields.
6. Use null values only if information cannot be found explicitly or implicitly.
7. Provide a brief summary after the JSON, including your reasoning for deduced values.
8. Prioritize information from amendments over the original agreement, especially time related information such as dates of signatures.
Critical Document Analysis:
- Construct a detailed timeline of the fund's lifecycle, including:
a) Initial Closing Date
b) First Closing Date
c) Final Closing Date (and any potential extensions)
d) Start and end of Investment Period
e) Total fund term and any extensions
- For each date or period, identify triggering events/conditions and calculate durations.
- Use the most recent applicable dates for calculations, particularly from amendments.
- Explain clauses allowing extensions or alterations of key dates/periods and their potential impact.
- Ensure all time-related terms in the JSON are filled with specific values, ranges, or calculated estimates.
- Break down the total fund term into its component periods (e.g., Investment Period, Follow-On Period, Remainder of Term).
- For interdependent dates, provide relationships in the JSON and explain calculations in the summary.
Document Interpretation Guidelines:
- Cross-reference relevant sections when interpreting complex terms or dates.
- Use financial expertise to infer logical connections between different terms and dates.
- Make reasonable inferences using industry knowledge and standard practices when information isn't explicitly stated.
- Quote relevant document sections to support interpretations.
- Consider impacts on fund structure and investor relationships.
- Ensure logical consistency between provided values.
- Pay attention to legal and financial terminology that may not exactly match JSON field names.
Additional Notes:
- The document text is OCR-processed; interpret unclear sections using context.
- Important information might be in unexpected places, including footnotes or definitions.
- Pay special attention to the last pages of the document which correspond with the images attached. In these images pay special attention to signatures and dates mentioned therein.
Document Date Hierarchy:
1. Always prioritize dates from signature pages over dates mentioned in the preamble or body of the agreement.
2. For critical dates like the First Close Date, the most recent date on any signature page typically supersedes earlier dates mentioned in the document.
3. Be aware that the date an agreement is drafted (often mentioned at the beginning) may differ significantly from when it's actually signed and executed.
Key Points for First Close Date:
- The First Close Date is the date the document is signed and executed + 1 day.
- This is usually found on the signature page.
Examples of thought processing:
If the text reads, "Return of Capital First, 100% to such Limited Partner until such Limited Partner has received distributions pursuant to this Section 7.4(a)(i) equal to such Limited Partner's aggregate Capital Contributions made in respect of the applicable Portfolio Investment," this implies a hurdle of 0%. Note this may not be in this document as this is just an example.
In your summary:
- Provide a clear, chronological explanation of the fund's timeline.
- Explain your reasoning for each calculated date and duration, citing relevant sections.
- Clarify any unclear or variable dates/periods and factors affecting them.
- Ensure consistency between the summary and the JSON.
- Explain calculations and assumptions for estimated dates.
- Break down the entire fund term, explaining how each period fits into the overall timeline.
"""
reflection_prompt = """
You are tasked with conducting a thorough review and verification of a limited partnership agreement document. A previous analysis has been performed, resulting in a JSON object and summary. Your job is to independently analyze the document, verify the existing information, identify any discrepancies or missing details, and provide a comprehensive, updated analysis.
Key Instructions:
1. Carefully read and interpret the entire limited partnership agreement document, including any amendments and signature pages.
2. Review the provided JSON object and summary from the previous analysis.
3. Conduct your own independent analysis of the document, paying attention to all relevant information that should be captured in the JSON structure.
4. Verify each piece of information in the existing JSON, updating or correcting as necessary.
5. Use your expertise in financial document analysis and understanding of limited partnership agreements throughout this task.
Document Analysis Guidelines:
1. Timeline Verification:
- Reconstruct the fund's lifecycle, including all key dates (Initial Closing, First Closing, Final Closing, etc.)
- Verify the calculation of the Investment Period and total fund term
- Ensure consistency between related dates and periods
- Pay special attention to signature pages for critical dates like the First Close Date
2. Financial Terms:
- Analyze hurdle rates, management fees, and carried interest calculations
- Verify any complex waterfall structures or distribution mechanisms
3. Investor Rights and Obligations:
- Review limited partner commitments and any special rights
- Check for key-person clauses or no-fault divorce provisions
4. Fund Structure and Governance:
- Verify details about general partners, management company, advisory boards, and investment committees
5. Investment Strategy and Restrictions:
- Review investment focus, geographical restrictions, and diversification requirements
- Check for investment limitations or concentration limits
6. Document Interpretation:
- Cross-reference relevant sections when interpreting complex terms or dates
- Make reasonable inferences using industry knowledge when information isn't explicitly stated
- Consider impacts on fund structure and investor relationships
- Ensure logical consistency between provided values
Output:
1. An updated JSON object with any necessary changes
2. A detailed analysis summary that includes:
- Confirmation of verified information
- Explanations for any changes or additions made to the JSON
- Discussion of any discrepancies found between the document and the previous analysis
- Identification of any new information not captured in the original JSON
- Explanation of any complex interpretations or inferences made
- Any ambiguities or unclear information in the document
- Suggestions for additional fields that might be valuable for future extractions
Additional Notes:
- The document text is OCR-processed; interpret unclear sections using context
- Important information might be in unexpected places, including footnotes or definitions
- Prioritize information from amendments over the original agreement, especially for time-related information
- Provide clear reasoning for any deduced values or interpretations
- Ensure all time-related terms in the JSON are filled with specific values, ranges, or calculated estimates
- Break down the total fund term into its component periods in your summary
Remember, your goal is to provide the most accurate and comprehensive analysis of the limited partnership agreement possible, building upon but not being constrained by the previous analysis.
"""
json_template = """
```json
{
"fundSize": {
"reportingCurrency": null,
"targetedAmount": null,
"LPMinimumCommit": null,
"maximumCap": null,
"firstCloseAmount": null,
"amountRaisedToDate": null,
"totalClosingsToDate": null,
},
"fundTerm": {
"firstCloseDate": null,
"targetedFinalCloseDate": null,
"investmentPeriod": null,
"followOnPeriod": null,
"remainderOfTerm": null,
"extensionPeriod": null,
},
"managementFees": {
"investmentPeriod": {
"percentage": null,
"basis": null,
},
"followOnPeriod": {
"percentage": null,
"basis": null,
},
"extensionPeriod": {
"percentage": null,
"basis": null,
}
},
"investmentFocus": {
"strategicFocus": null,
"geographicFocus": null,
"industryFocus": null,
},
"miscTerms": {
"reportingCurrency": null,
"hurdleRate": null,
"carriedInterest": null,
"carryCatchUp": null,
"carryEscrow": null,
"gpCommit": null
}
}
```
"""

57
task/requirements.txt Normal file
View File

@ -0,0 +1,57 @@
aiohappyeyeballs==2.4.0
aiohttp==3.10.5
aiosignal==1.3.1
annotated-types==0.7.0
anyio==4.6.0
async-timeout==4.0.3
attrs==24.2.0
boto3==1.35.11
botocore==1.35.11
certifi==2024.8.30
cffi==1.17.1
charset-normalizer==3.3.2
cryptography==43.0.1
et-xmlfile==1.1.0
exceptiongroup==1.2.2
frozenlist==1.4.1
greenlet==3.1.1
h11==0.14.0
httpcore==1.0.5
httpx==0.27.2
idna==3.10
jmespath==1.0.1
jsonpatch==1.33
jsonpointer==3.0.0
multidict==6.1.0
numpy==1.26.4
openpyxl==3.1.5
orjson==3.10.7
packaging==24.1
pandas==2.2.3
pdfminer.six==20231228
pdfplumber==0.11.4
pillow==10.4.0
pycparser==2.22
pydantic==2.9.2
pydantic_core==2.23.4
PyPDF2==3.0.1
pypdfium2==4.30.0
python-dateutil==2.9.0.post0
pytz==2024.2
PyYAML==6.0.2
requests==2.32.3
s3transfer==0.10.2
six==1.16.0
sniffio==1.3.1
SQLAlchemy==2.0.35
temp==2020.7.2
tenacity==8.5.0
timedelta==2020.12.3
typing_extensions==4.12.2
tzdata==2024.1
urllib3==2.2.2
xlrd==2.0.1
yarl==1.11.1
# Google Gemini
google-genai>=1.0.0

63
task/s3Ops.py Normal file
View File

@ -0,0 +1,63 @@
import boto3
import pandas as pd
from io import BytesIO
# from dotenv import load_dotenv
#
# load_dotenv()
def list_files_in_s3_folder(bucket_name, folder_path):
"""
List all files in a specified S3 folder.
Args:
bucket_name (str): The name of the S3 bucket.
folder_path (str): The path to the folder in the S3 bucket.
Returns:
list: A list of file names in the specified S3 folder.
"""
s3_client = boto3.client('s3')
# List objects in the specified S3 folder
result = s3_client.list_objects_v2(Bucket=bucket_name, Prefix=folder_path)
if 'Contents' not in result:
return []
# Extract the file names
file_names = [item['Key'] for item in result['Contents']]
return file_names
def read_file_from_s3(bucket_name, file_key):
"""
Read the content of a file from S3 and return it as a byte stream.
Args:
bucket_name (str): The name of the S3 bucket.
file_key (str): The key (path) to the Excel file in the S3 bucket.
Returns:
BytesIO: The content of the file as a BytesIO.
"""
s3_client = boto3.client('s3')
try:
# Download the file content from S3
response = s3_client.get_object(Bucket=bucket_name, Key=file_key)
# Read the file content into a pandas DataFrame
file_content = response['Body'].read()
# Use BytesIO to read the file content in-memory
s3_file = BytesIO(file_content)
return s3_file
except Exception as e:
print(f"Error reading file from S3: {e}")
return None

91
task/sqs_processor.py Normal file
View File

@ -0,0 +1,91 @@
import os
import json
import time
import boto3
import logging
from botocore.exceptions import ClientError
from lambda_function import lambda_handler
def read_sqs_messages():
"""Read messages from SQS queue"""
# Get queue URL from environment variable or use a default for testing
queue_url = os.environ.get('SQS_QUEUE_URL')
try:
# Create SQS client
sqs = boto3.client('sqs')
while True:
try:
# Receive message from SQS queue
response = sqs.receive_message(
QueueUrl=queue_url,
AttributeNames=['All'],
MaxNumberOfMessages=1,
MessageAttributeNames=['All'],
VisibilityTimeout=30,
WaitTimeSeconds=20
)
if 'Messages' in response:
messages = response['Messages']
print(f"Received {len(messages)} messages")
for message in messages:
receipt_handle = message['ReceiptHandle']
message_body = message['Body']
print("Message Body")
print(message_body)
try:
# Process the message with error handling
lambda_handler(message_body)
# Delete the message from the queue only if processing succeeded
sqs.delete_message(
QueueUrl=queue_url,
ReceiptHandle=receipt_handle
)
print(f"Successfully processed and deleted message: {receipt_handle}")
except (KeyError, json.JSONDecodeError, TypeError) as e:
# Log the error and skip this record
print(f"ERROR: Failed to process message due to data/parsing error: {e}")
print(f"Skipping problematic message: {message_body[:500]}") # Log first 500 chars
# Delete the problematic message to prevent infinite retries
sqs.delete_message(
QueueUrl=queue_url,
ReceiptHandle=receipt_handle
)
print(f"Deleted problematic message from queue to prevent pod restart")
except Exception as e:
# For other errors, log but don't delete - let SQS retry later
print(f"ERROR: Unexpected error processing message: {e}")
print(f"Message will be retried after visibility timeout")
# Don't delete the message, let it become visible again for retry
else:
print("No messages available in the queue")
except ClientError as e:
print(f"Error processing SQS messages: {e}")
time.sleep(5) # Wait before retrying
except Exception as e:
print(f"Fatal error: {e}")
raise
if __name__ == "__main__":
print("SQS message reader starting")
try:
read_sqs_messages()
except KeyboardInterrupt:
print("SQS message reader stopped by user")
except Exception as e:
print(f"SQS reader failed with error: {e}")

243
task/tester.py Normal file
View File

@ -0,0 +1,243 @@
DOCUMENT_CLASSIFY_PROMPT = """
You are an expert in document classification where you are given a document under <document> xml tags and you need to classify it based on the data inside <document-type> xml tag.
Your output should be enclosed inside <output> xml tag and it should be based on one of the <document-type> options mentioned below.Just output <document-type> in the output <output> xml tag.
### Document
<document>
{document}
</document>
###
## Document Types that need to classified.
<document-type>
["LPA(Limited Partner Agreement)", "LPA-Amendment(Limited Partner Agreement-Amendment)", "Side Letter"]
</document-type>
"""
LPA_STATEMENT_PROMPT = """
<SYSTEM>
You are an expert in LPA (Limited Partner Agreement) data extraction that uses a Chain of Thought (CoT) approach with reflection to answer queries. Follow these steps:
1. Think through the problem step by step within the <thinking> tags.
2. Reflect on your thinking to check for any errors or improvements within the <reflection> tags.
3. Make any necessary adjustments based on your reflection.
4. Provide your final, concise answer within the <output> tags.
Important: The <thinking> and <reflection> sections are for your internal reasoning process only.
Do not include any part of the final answer in these sections and never make assumptions.
The actual response to the query must be entirely contained within the <output> tags.
Use the following format for your response:
<thinking>
[Your step-by-step reasoning goes here. This is your internal thought process, not the final answer. Please explain the reason for selecting each piece of information.]
</thinking>
<reflection>
[Your reflection on your reasoning, checking for errors or improvements]
</reflection>
<output>
[Your final, concise answer to the query in the specified JSON format. This is the only part that will be shown to the user.]
</output>
</SYSTEM>
<USER>
You are given data in the <document> xml tag. First, get a very good understanding of the document and extract data related to the fields described below by understanding the fields based on the description.
### Document
<document>
{document}
</document>
Final output to place under <output> xml tag and populate empty values if the data is not present in the document.
<output-format>
{{
"fundName": "name of the fund",
"fileInfo":"name of the fund and under which type of the document is it - Date of the agreement"
"managementFee": "Extract the following information from the document:
1. Annual management fee percentage(s), including any changes over time
2. Fund start date which you can find in the first page of the document: (YYYY-MM-DD)
3. Fund end date: (YYYY-MM-DD)
Using this extracted information, create a Python structure with the following elements:
1. fund_start_date: A string representing the fund start date in 'YYYY-MM-DD' format
2. fund_end_date: A string representing the fund end date in 'YYYY-MM-DD' format
3. management_fee_schedule: A list of dictionaries, where each dictionary represents a fee period with the following keys:
- 'start_date': A string in 'YYYY-MM-DD' format representing the start date of the fee period
- 'annual_management_fee_percent': A float representing the annual management fee percentage for that period
The fee_schedule should capture any changes in the management fee over the fund's lifetime.
Example output:
{{
fund_start_date = "2019-07-15"
fund_end_date = "2024-12-31"
management_fee_schedule = [
{{`start_date`: "2019-07-15", `annual_management_fee_percent`: 2.5}},
{{`start_date`: "2023-07-01", `annual_management_fee_percent`: 2.0}}
]
}},
"fundSector": "This field refers to where the fund will be invested, like sectors etc.",
"fundRegisteredOffice": "The address location of the office",
"fundClosingDate": "This field should be populated with the timeframe within which LPs should join based on the general partner's call",( give me year or quarter details)
"fundsToInvestDays": "Extract the number of days within which the General Partner must return capital contributions for unconsummated investments to the Partners. Please provide only the numerical value?",
"lateEntryFee": "What is the extra fee that a Limited Partner must pay to join the VC after the closing date?",
"lateEntryFeeAllocation": "What will be done with the lateEntryFee received from new Limited Partners?",
"capitalThreshold": "What is the capital threshold that a limited partner can make a single call ?",
"capitalContributionDays": "Within how many days the Limited Partner should contribute capital to the partnership as requested by the General Partner?((Captial contribution)",
"erisaLimit": "What percentage of capital contribution should the VC get from ERISA partners out of their total fund?",
"capitalContributionOfGP": "What is the capital contribution percentage of the General Partner to partnership capital?",
"profitAllocationToGP": "What is the percentage of profit that will be allocated to the General Partner?", # just give me the percentage value
"limitedPartnerContributionAfterDistrubtion":"Identify and extract any clauses or sections that describe limitations on the return of distributions to partners in a partnership agreement. Specifically, look for:
1. The maximum amount partners may be required to return
2. Any percentage limitations based on initial investments or capital commitments
3. Distinctions between different types of partners (e.g., Limited Partners vs. General Partners)
4. Time limits for requesting the return of distributions
5. Conditions under which distributions may be recalled
6. Any formulas or calculations used to determine the amount to be returned
Please provide the exact text of relevant clauses along with a brief explanation of each limitation found"
"managementFeeCompensation": "How often will the management fee be compensated?",
"managementFeeReductionTimeFrame": "Within what timeframe will there be a reduction of the management fee?",
"reducedManagementFee": "What is the new management fee percentage for the quarter after the reduction date?",
"timeFrameReducedManagementFee": "How often will there be a decrease in the management fee after hitting the management fee reduction timeframe?",# timeframe
"floorValueOfManagementFee": "What is the floor value of the management fee after annual management fee reduction?",
"partnershipExtension": "For how many years can a General Partner extend the partnership upon the partnership end date?",
"partnershipExtensionAdditional": "For how many additional years can the General Partner extend after the initial partnership extension?",
"Management Expenses":"Extract complete information in any clauses or sections that describe about the expenses related to management and there limitations and i want the section as well",
"Fund/Partnership Expenses": "Extract complete information in any clauses or sections that describe about the expenses related to Fund/Partnership and there limitations and i want the section as well",
"OnetimeOrganizationExpenses":"Extract complete information in any clauses or sections that describe about the expenses related to OnetimeOrganization and there limitations and i want the section as well"
}}
</output-format>
</user>
"""
SIDE_LETTER_PROMPT = """
You are given side letter document related to a limited partner under <document> xml tag.
First get a very good understanding of the below document under <document> xml tag . Your task is to generate the response in below format <output-format> in <output> xml tags and partner id related to the partner to be written to <id>
## Document
<document>
{document}
</document>
## partner id document
<partner-id-document>
{role_doc}
</partner-id-document>
<output-format>
{{
"fundName":"Name of the fund",
"fundDate":"Limited Partner Agreement date in format : YYYY-MM-DD",
"partnerName":"Name of the investing partner",
"quarter":"",
"platform":"Name of the platform which generated the report,if you are not sure return empty",
"carriedInterest": " <case:1> : If there is a change in carried interest then execute below one
Just give me the percentage value of profit/carry interest that will be allocated to the General Partner.
else
<case:2>: Just return `No changes` ",
"managementFee": "
<case:1> : If there is a changing in management fee structure execute below one
Extract the following information from the document:
1. Annual management fee percentage(s), including any changes over time
2. Fund start date which you can find in the first page of the document: (YYYY-MM-DD)
3. Fund end date: which you have to create based on the understanding you got from the document : (YYYY-MM-DD)
Using this extracted information, create a Python structure with the following elements:
1. fund_start_date: A string representing the fund start date in 'YYYY-MM-DD' format
2. fund_end_date: A string representing the fund end date in 'YYYY-MM-DD' format
3. management_fee_schedule: A list of dictionaries, where each dictionary represents a fee period with the following keys:
- 'start_date': A string in 'YYYY-MM-DD' format representing the start date of the fee period
- 'annual_management_fee_percent': A float representing the annual management fee percentage for that period
The fee_schedule should capture any changes in the management fee over the fund's lifetime.
Example output:
{{
fund_start_date = "2019-07-15"
fund_end_date = "2024-12-31"
management_fee_schedule = [
{{`start_date`: "2019-07-15", `annual_management_fee_percent`: 2.5}},
{{`start_date`: "2023-07-01", `annual_management_fee_percent`: 2.0}}
]
}}
else
<case:2> :Just return `No changes` ",
"sideLetterKeyPoints": "Extract key details in the document, Just return it in a string",
}}
</output-format>
<id>
"Extract value related to partner id name from <role-document> document"
</id>
"""
import pdfplumber
with pdfplumber.open('Side_letter_Greylock Partners_062723.pdf') as pdf:
list_pages = []
for page in pdf.pages:
text = page.extract_text()
if text:
list_pages.append(text.strip())
pdf_string = "".join(list_pages)
data_string = """"data": [
{
"label": "FIN CAP INVEST LLC",
"value": "66fbfbd920655b335e6d1ccc"
},
{
"label": "Avinash",
"value": "6740785c268e6da0e1c746e6"
},
{
"label": "CODY HEALTHCARE S CORP",
"value": "66fbfbd820655b335e6d1ca8"
},
{
"label": "Enterprise International, Inc.",
"value": "66fbfbd820655b335e6d1cc0"
},
{
"label": "Pradeepp Kukunuri",
"value": "66fbfbc6f78cfc9f3cafb51c"
},
{
"label": "Shresth",
"value": "672dd65e6844833570e5b0e2"
},
{
"label": "Support",
"value": "672dd6376844833570e5b0be"
},
{
"label": "Greylock Partners",
"value": "66fbfbd820655b335e6d1cb4"
}
]"""
from llm_bedrock import model
import re
output = model.invoke(SIDE_LETTER_PROMPT.format(document=pdf_string,role_doc=data_string)).content
pattern = r'<output>(.*?)</output>'
match = re.search(pattern, output, re.DOTALL)
print(match.group(1).strip())
id = r'<id>(.*?)</id>'
match = re.search(id,output,re.DOTALL)
print(match.group(1).strip())
print(match)

244
task/utils/account_type.py Normal file
View File

@ -0,0 +1,244 @@
account_type_data = {
"Bank": "Bank",
"Capital Call Receivable - LP": "Capital Call Receivable - LP",
"Portfolio Interest Receivable": "Portfolio Interest Receivable",
"Portfolio interest receivable": "Portfolio Interest Receivable",
"Portfolio interest income": "Portfolio Interest Income",
"Due to related parties": "Due to Related Party",
"Due to investment": "Due to investment",
"Due to management company": "Due to Management Company",
"Investment - unrealized gain/loss": "Investment - Unrealized Gain/Loss",
"Unrealized gain/loss": "Unrealized Gain/Loss",
"Portfolio company receivable": "Portfolio Company Receivable",
"Realized gain/loss": "Realized Gain/Loss",
"Distribution payable": "Distribution Payable",
"Cash distribution - LP": "Cash Distribution - LP",
"Due to general partner": "Due to General Partner",
"Other prepaids": "Other Prepaids",
"Portfolio Company Receivable": "Portfolio Company Receivable",
"Misc Receivable": "Misc Receivable",
"Misc receivable": "Misc Receivable",
"Temporarily Commingled Cash from Bank Receivable":
"Temporarily Commingled Cash from Bank Receivable",
"Due from the Management Company": "Due from the Management Company",
"Receivable between Master Feeder": "Receivable between Master Feeder",
"Receivable from Affiliate": "Receivable from Affiliate",
"Payable between Master Feeder": "Payable between Master Feeder",
"Payable to Affiliate": "Payable to Affiliate",
"Due from Affiliates": "Due from Affiliates",
"Due to Affiliates": "Due to Affiliates",
"Other Prepaids": "Other Prepaids",
"Accelaration Prepaids": "Accelaration Prepaids",
"Organization costs asset": "Organization Costs Asset",
"Organization Costs Asset": "Organization Costs Asset",
"Investment": "Investment",
"Investment - Unrealized Gain/Loss": "Investment - Unrealized Gain/Loss",
"Due to Management Company": "Due to Management Company",
"Management Fee Payable": "Management Fee Payable",
"Admission Fee Payable": "Admission Fee Payable",
"Admission Fees": "Admission Fees",
"Due to General Partner": "Due to General Partner",
"Due to Limited Partner": "Due to Limited Partner",
"Management Fees Prepaid": "Management Fees Prepaid",
"Due to Portfolio Company": "Due to Portfolio Company",
"Due to Other": "Due to Other",
"Due to Related Party": "Due to Related Party",
"Deferred Capital Calls - LP": "Deferred Capital Calls - LP",
"Deferred Capital Calls - GP": "Deferred Capital Calls - GP",
"Deferred capital calls - GP": "Deferred Capital Calls - GP",
"Audit Accrual": "Audit Accrual",
"Fund Administration Accrual": "Fund Administration Accrual",
"Tax Accrual": "Tax Accrual",
"Other Accrual": "Other Accrual",
"Syndication Costs - General": "Syndication Costs - General",
"Contributed Capital - LP": "Contributed Capital - LP",
"Misc Income": "Misc Income",
"Portfolio Interest Income": "Portfolio Interest Income",
"Unrealized Gain/Loss": "Unrealized Gain/Loss",
"Management Fees": "Management Fees",
"Audit Fees": "Audit Fees",
"Tax Prep Fees": "Tax Prep Fees",
"Fund Administration Fees": "Fund Administration Fees",
"Other assets": "Other assets",
"Due to limited partner": "Due to Limited Partner",
"Administration Fees": "Administration Fees",
"Administration Fees - Allocated from Master":
"Administration Fees - Allocated from Master",
"Audit Fees - Allocated from Master": "Audit Fees - Allocated from Master",
"Bank Charges - Allocated from Master":
"Bank Charges - Allocated from Master",
"Commission Expense": "Commission Expense",
"Commission Expense - Allocated from Master":
"Commission Expense - Allocated from Master",
"FATCA Expenses": "FATCA Expenses",
"FATCA Expenses - Allocated from Master":
"FATCA Expenses - Allocated from Master",
"FS Preparation": "FS Preparation",
"FS Preparation - Allocated From Master":
"FS Preparation - Allocated From Master",
"Legal Expenses - Allocated from Master":
"Legal Expenses - Allocated from Master",
"Marketing Expenses - Allocated from Master":
"Marketing Expenses - Allocated from Master",
"Offering & Organizational Cost": "Offering & Organizational Cost",
"Offering & Organizational Cost - Allocated from Master":
"Offering & Organizational Cost - Allocated from Master",
"Performance Fees": "Performance Fees",
"Prepaid Offering & Organizational Cost":
"Prepaid Offering & Organizational Cost",
"Travelling Expense": "Travelling Expense",
"Travelling Expense - Allocated from Master":
"Travelling Expense - Allocated from Master",
"Administration Fees Payable": "Administration Fees Payable",
"Bank Charges Payable": "Bank Charges Payable",
"FATCA Payable": "FATCA Payable",
"FS Prepartion Payable": "FS Prepartion Payable",
"Marketing Expenses Payable": "Marketing Expenses Payable",
"Offering & Organizational Cost Payable":
"Offering & Organizational Cost Payable",
"Performance Fees Payable": "Performance Fees Payable",
"Travelling Expense Payable": "Travelling Expense Payable",
"Reimbursement Payable": "Reimbursement Payable",
"Audit Fees Payable": "Audit Fees Payable",
"Due to/from GP": "Due to/from GP",
"Unrealized Gain (Loss) from Private Investment Companies":
"Unrealized Gain (Loss) from Private Investment Companies",
"Unrealized Gain (Loss) from Private Investment Companies - Allocated from Master":
"Unrealized Gain (Loss) from Private Investment Companies - Allocated from Master",
"Unrealized Gain (Loss) from Cryptocurrencies (Price) - Allocated from Master":
"Unrealized Gain (Loss) from Cryptocurrencies (Price) - Allocated from Master",
"Unrealized Gain (Loss) on Investment in Cryptocurrencies (Price)":
"Unrealized Gain (Loss) on Investment in Cryptocurrencies (Price)",
"Due from Limited Partner": "Due from Limited Partner",
"Due/To from Offshore Feeder Fund": "Due/To from Offshore Feeder Fund",
"Due/To from Offshore Master Fund": "Due/To from Offshore Master Fund",
"Due/To from Onshore Feeder Fund": "Due/To from Onshore Feeder Fund",
"Receivable from pending settlement": "Receivable from pending settlement",
"Investment in Cryptocurrencies, (At Cost)":
"Investment in Cryptocurrencies, (At Cost)",
"Investment in Fixed Income Bond": "Investment in Fixed Income Bond",
"Investment in Master": "Investment in Master",
"Investment in Private Investment Companies":
"Investment in Private Investment Companies",
"Investment In Safes, (At Cost)": "Investment In Safes, (At Cost)",
"Investment In SAFT, (At Cost)": "Investment In SAFT, (At Cost)",
"Change in Unrealized Gain (Loss) from Private Investment Companies":
"Change in Unrealized Gain (Loss) from Private Investment Companies",
"Change in Unrealized Gain (Loss) on Investment in Cryptocurrencies (Price)":
"Change in Unrealized Gain (Loss) on Investment in Cryptocurrencies (Price)",
"Change in Unrealized Gain (Loss) on Investment in Cryptocurrencies (Price) - Allocated from Master":
"Change in Unrealized Gain (Loss) on Investment in Cryptocurrencies (Price) - Allocated from Master",
"Income from Master": "Income from Master",
"Staking Income": "Staking Income",
"Staking Income - Allocated from Master":
"Staking Income - Allocated from Master",
"Interest Income - Allocated from Master":
"Interest Income - Allocated from Master",
"Realized Gain (Loss) on Investment in Cryptocurrencies (Price)":
"Realized Gain (Loss) on Investment in Cryptocurrencies (Price)",
"Realized Gain (Loss) on Investment in Cryptocurrencies (Price) - Allocated from Master":
"Realized Gain (Loss) on Investment in Cryptocurrencies (Price) - Allocated from Master",
"Legal Fees": "Legal Fees",
"Legal Fees Accrual": "Legal Fees Accrual",
"Interest Payable": "Interest Payable",
"Loan Payable": "Loan Payable",
"Organization Costs Expense": "Organization Costs Expense",
"Other Professional Fees": "Other Professional Fees",
"Filing Fees": "Filing Fees",
"Other Fees": "Other Fees",
"Office supplies": "Office supplies",
"Organization costs amortization": "Organization Costs Amortization",
"Distribution Payable": "Distribution Payable",
"Distribution Payable - GP": "Distribution Payable - GP",
"Tax Preparation Fees Accrual": "Tax Preparation Fees Accrual",
"Realized Gain/Loss": "Realized Gain/Loss",
"Escrow from portfolio investments sold":
"Escrow from portfolio investments sold",
"Cash Distribution - LP": "Cash Distribution - LP",
"Cash Distribution - GP": "Cash Distribution - GP",
"Cash distribution - GP": "Cash Distribution - GP",
"Distribution payable - GP": "Distribution Payable - GP",
"Cash Carry Distributions": "Cash Carry Distributions",
"Carried Interest Distribution - GP": "Carried Interest Distribution - GP",
"Bank 1099able": "Bank 1099able",
"Audit fees": "Audit Fees",
"Bank charges": "Bank Charges",
"Bank Charges": "Bank Charges",
"Bank interest income": "Bank Interest Income",
"Investment - capitalized interest": "Investment",
"Capital call receivable - LP": "Capital Call Receivable - LP",
"Misc income": "Misc Income",
"Prepaid management fees": "Management Fees Prepaid",
"Consulting": "Consulting",
"Organization expenses": "Organization Expenses",
"Organization Expenses": "Organization Expenses",
"Due diligence": "Due Diligence",
"Due Diligence": "Due Diligence",
"Interest expense": "Interest Expense",
"Interest Expense": "Interest Expense",
"Contributed capital - LP": "Contributed Capital - LP",
"Deal fees": "Deal Fees",
"Deal Fees": "Deal Fees",
"Deferred capital calls - LP": "Deferred Capital Calls - LP",
"Due from general partner": "Due from General Partner",
"Due from General Partner": "Due from General Partner",
"Due from related parties": "Due from Related Parties",
"Due from the management company": "Due from the Management Company",
"Due to other": "Due to Other",
"Due to portfolio company": "Due to Portfolio Company",
"Dues and subscriptions": "Dues and Subscriptions",
"Filing fees": "Filing Fees",
"Fund administration fees": "Fund Administration Fees",
"Legal fees": "Legal Fees",
"Legal accrual": "Legal Fees Accrual",
"Management fee payable": "Management Fee Payable",
"Management fees": "Management Fees",
"Management fees offset": "Management fees offset",
"Other accrual": "Other Accrual",
"Taxes": "Taxes",
"Conference and trade shows": "Conference and Trade Shows",
"Website design": "Website Design",
"Tax prep fees": "Tax Prep Fees",
"Syndication costs - Placement agent fees":
"Syndication Costs - Placement Agent Fees",
"Software": "Software",
"Reimbursement of partnership expenses":
"Reimbursement of Partnership Expenses",
"Reimbursement of Partnership Expenses":
"Reimbursement of Partnership Expenses",
"Other interest income": "Other Interest Income",
"Other Interest Income": "Other Interest Income",
"Fund administration accrual": "Fund Administration Accrual",
"Contributed capital - GP": "Contributed Capital - GP",
"Contributed Capital - GP": "Contributed Capital - GP",
"Contributed capital - cashless offset - GP":
"Contributed capital - cashless offset - GP",
"Capital call receivable - GP": "Capital Call Receivable - GP",
"Capital Call Receivable - GP": "Capital Call Receivable - GP",
"Tax preparation fees accrual": "Tax Preparation Fees Accrual",
"Bank - Checking Account": "Bank",
"Bank 1099able - Checking Account": "Bank",
"Money market funds": "Money market funds",
"Expense Payable": "Expense Payable",
"Audit accrual": "Audit Accrual",
"Other professional fees": "Other Professional Fees",
"Bank 1099able - Checking Account - check":
"Bank 1099able - Checking Account",
"Organization costs expense": "Organization Costs Expense",
"Marketing expenses": "Marketing Expenses",
"Marketing Expenses": "Marketing Expenses",
"Travel": "Travel",
"Meal": "Meal",
"Syndication costs - general": "Syndication Costs - General",
"Other fees": "Other Fees",
"Subscription Fee - LP": "Subscription Fee - LP",
"Organization Costs Amortization": "Organization Costs Amortization",
"Acquisition fee": "Acquisition Fee",
"Acquisition Fee": "Acquisition Fee",
"Bank Interest Income": "Bank Interest Income",
"Interest Income": "Interest Income",
"Late Interest Income": "Late Interest Income",
"Dividend Income": "Dividend Income",
"Other Income": "Other Income",
"Other professional Fees": "Other Professional Fees",
}

434
task/utils/prompts.py Normal file
View File

@ -0,0 +1,434 @@
DOCUMENT_CLASSIFY_PROMPT = """
You are an expert in document classification where you are given a document under <document> xml tags and you need to classify it based on the data inside <document-type> xml tag.
Your output should be enclosed inside <output> xml tag and it should be based on one of the <document-type> options mentioned below.Just output <document-type> in the output <output> xml tag.
### Document
<document>
{document}
</document>
###
## Document Types that need to classified.
<document-type>
["LPA(Limited Partner Agreement)", "LPA-Amendment(Limited Partner Agreement-Amendment)", "Side Letter"]
</document-type>
"""
LPA_STATEMENT_PROMPT = """
<SYSTEM>
You are an expert in LPA (Limited Partner Agreement) data extraction that uses a Chain of Thought (CoT) approach with reflection to answer queries. Follow these steps:
1. Think through the problem step by step within the <thinking> tags.
2. Reflect on your thinking to check for any errors or improvements within the <reflection> tags.
3. Make any necessary adjustments based on your reflection.
4. Provide your final, concise answer within the <output> tags.
Important: The <thinking> and <reflection> sections are for your internal reasoning process only.
Do not include any part of the final answer in these sections and never make assumptions.
The actual response to the query must be entirely contained within the <output> tags.
Use the following format for your response:
<thinking>
[Your step-by-step reasoning goes here. This is your internal thought process, not the final answer. Please explain the reason for selecting each piece of information.]
</thinking>
<reflection>
[Your reflection on your reasoning, checking for errors or improvements]
</reflection>
<output>
[Your final, concise answer to the query in the specified JSON format. This is the only part that will be shown to the user.]
</output>
</SYSTEM>
<USER>
You are given data in the <document> xml tag. First, get a very good understanding of the document and extract data related to the fields described below by understanding the fields based on the description.
### Document
<document>
{document}
</document>
Final output to place under <output> xml tag and populate empty values if the data is not present in the document.
<output-format>
{{
"fundName": "name of the fund",
"fileInfo":"name of the fund and under which type of the document is it - Date of the agreement"
"managementFee": "Extract the following information from the document:
1. Annual management fee percentage(s), including any changes over time
2. Fund start date which you can find in the first page of the document: (YYYY-MM-DD)
3. Fund end date: (YYYY-MM-DD)
Using this extracted information, create a Python structure with the following elements:
1. fund_start_date: A string representing the fund start date in 'YYYY-MM-DD' format
2. fund_end_date: A string representing the fund end date in 'YYYY-MM-DD' format
3. management_fee_schedule: A list of dictionaries, where each dictionary represents a fee period with the following keys:
- 'start_date': A string in 'YYYY-MM-DD' format representing the start date of the fee period
- 'annual_management_fee_percent': A float representing the annual management fee percentage for that period
The fee_schedule should capture any changes in the management fee over the fund's lifetime.
Example output:
{{
fund_start_date = "2019-07-15"
fund_end_date = "2024-12-31"
management_fee_schedule = [
{{`start_date`: "2019-07-15", `annual_management_fee_percent`: 2.5}},
{{`start_date`: "2023-07-01", `annual_management_fee_percent`: 2.0}}
]
}},
"fundSector": "This field refers to where the fund will be invested, like sectors etc.",
"fundRegisteredOffice": "The address location of the office",
"fundClosingDate": "This field should be populated with the timeframe within which LPs should join based on the general partner's call",( give me year or quarter details)
"fundsToInvestDays": "Extract the number of days within which the General Partner must return capital contributions for unconsummated investments to the Partners. Please provide only the numerical value?",
"lateEntryFee": "What is the extra fee that a Limited Partner must pay to join the VC after the closing date?",
"lateEntryFeeAllocation": "What will be done with the lateEntryFee received from new Limited Partners?",
"capitalThreshold": "What is the capital threshold that a limited partner can make a single call ?",
"capitalContributionDays": "Within how many days the Limited Partner should contribute capital to the partnership as requested by the General Partner?((Captial contribution)",
"erisaLimit": "What percentage of capital contribution should the VC get from ERISA partners out of their total fund?",
"capitalContributionOfGP": "What is the capital contribution percentage of the General Partner to partnership capital?",
"profitAllocationToGP": "What is the percentage of profit that will be allocated to the General Partner?", # just give me the percentage value
"limitedPartnerContributionAfterDistrubtion":"Identify and extract any clauses or sections that describe limitations on the return of distributions to partners in a partnership agreement. Specifically, look for:
1. The maximum amount partners may be required to return
2. Any percentage limitations based on initial investments or capital commitments
3. Distinctions between different types of partners (e.g., Limited Partners vs. General Partners)
4. Time limits for requesting the return of distributions
5. Conditions under which distributions may be recalled
6. Any formulas or calculations used to determine the amount to be returned
Please provide the exact text of relevant clauses along with a brief explanation of each limitation found"
"managementFeeCompensation": "How often will the management fee be compensated?",
"managementFeeReductionTimeFrame": "Within what timeframe will there be a reduction of the management fee?",
"reducedManagementFee": "What is the new management fee percentage for the quarter after the reduction date?",
"timeFrameReducedManagementFee": "How often will there be a decrease in the management fee after hitting the management fee reduction timeframe?",# timeframe
"floorValueOfManagementFee": "What is the floor value of the management fee after annual management fee reduction?",
"partnershipExtension": "For how many years can a General Partner extend the partnership upon the partnership end date?",
"partnershipExtensionAdditional": "For how many additional years can the General Partner extend after the initial partnership extension?",
"Management Expenses":"Extract complete information in any clauses or sections that describe about the expenses related to management and there limitations and i want the section as well",
"Fund/Partnership Expenses": "Extract complete information in any clauses or sections that describe about the expenses related to Fund/Partnership and there limitations and i want the section as well",
"OnetimeOrganizationExpenses":"Extract complete information in any clauses or sections that describe about the expenses related to OnetimeOrganization and there limitations and i want the section as well"
}}
</output-format>
</user>
"""
SIDE_LETTER_PROMPT = """
You are given side letter document related to a limited partner under <document> xml tag.
First get a very good understanding of the below document under <document> xml tag . Your task is to generate the response in below format <output-format> in <output> xml tags and partner id related to the partner to be written to <id>
## Document
<document>
{document}
</document>
## partner id document
<partner-id-document>
{role_doc}
</partner-id-document>
<output-format>
{{
"fundName":"Name of the fund",
"fundDate":"Limited Partner Agreement date in format : YYYY-MM-DD",
"partnerName":"Name of the investing partner",
"quarter":"",
"platform":"Name of the platform which generated the report,if you are not sure return empty",
"carriedInterest": " <case:1> : If there is a change in carried interest then execute below one
Just give me the percentage value of profit/carry interest that will be allocated to the General Partner.
else
<case:2>: Just return `No changes` ",
"managementFee": "
<case:1> : If there is a changing in management fee structure execute below one
Extract the following information from the document:
1. Annual management fee percentage(s), including any changes over time
2. Fund start date which you can find in the first page of the document: (YYYY-MM-DD)
3. Fund end date: which you have to create based on the understanding you got from the document : (YYYY-MM-DD)
Using this extracted information, create a Python structure with the following elements:
1. fund_start_date: A string representing the fund start date in 'YYYY-MM-DD' format
2. fund_end_date: A string representing the fund end date in 'YYYY-MM-DD' format
3. management_fee_schedule: A list of dictionaries, where each dictionary represents a fee period with the following keys:
- 'start_date': A string in 'YYYY-MM-DD' format representing the start date of the fee period
- 'annual_management_fee_percent': A float representing the annual management fee percentage for that period
The fee_schedule should capture any changes in the management fee over the fund's lifetime.
Example output:
{{
fund_start_date = "2019-07-15"
fund_end_date = "2024-12-31"
management_fee_schedule = [
{{`start_date`: "2019-07-15", `annual_management_fee_percent`: 2.5}},
{{`start_date`: "2023-07-01", `annual_management_fee_percent`: 2.0}}
]
}}
else
<case:2> :Just return `No changes` ",
"sideLetterKeyPoints": "Extract key details in the document, Just return it in a string",
}}
</output-format>
<id>
"Extract value related to partner id name from <role-document> document"
</id>
"""
ZIVE_SECURITY_TYPE_PROMPT = """
You are Fund AI assistant . Your task is to convert "Security type" fund investment to platform supported security type. If you are unsure about the security type make "PREFERRED" as default security type.
## Below are the security type that platform supported:
<securities>
<security>
<type>PRICED</type>
<description>A security issued during a funding round where the company's valuation has been established. Investors purchase shares at a fixed price based on the agreed pre-money valuation.</description>
</security>
<security>
<type>PRE MONEY SAFE</type>
<description>A SAFE is a contract that allows investors to provide capital to a company in exchange for the right to receive equity in the future. A pre-money SAFE sets the valuation cap before new investment, not including the incoming SAFE money.</description>
</security>
<security>
<type>CONVERTIBLE NOTE</type>
<description>A type of debt instrument that converts into equity upon a specific event, such as a future financing round. Convertible notes often include interest and may have a valuation cap or discount for conversion.</description>
</security>
<security>
<type>WARRANTS</type>
<description>A financial instrument that gives the holder the right to purchase company stock at a specific price before expiration. Warrants are often issued as part of a funding deal or incentive.</description>
</security>
<security>
<type>PREFERRED</type>
<description>Preferred stock offers investors certain privileges over common stockholders, such as priority in dividends and liquidation preferences. It is commonly issued during VC funding rounds.</description>
</security>
<security>
<type>TOKEN</type>
<description>A blockchain-based digital asset representing various rights or utilities within a project, such as governance, usage, or financial stake. Tokens are issued during ICOs (Initial Coin Offerings) or similar fundraising efforts.</description>
</security>
<security>
<type>COMMON</type>
<description>Common stock represents basic ownership in a company, typically held by founders and employees. Common stockholders usually have voting rights but are last in line for payouts during liquidation.</description>
</security>
<security>
<type>OPTIONS</type>
<description>Stock options grant the holder the right to purchase shares at a pre-determined price within a set time frame. Often used to incentivize employees or early contributors.</description>
</security>
<security>
<type>BRIDGE LOAN</type>
<description>Short-term debt financing provided to a company to cover immediate capital needs until a more significant funding round or event occurs.</description>
</security>
<security>
<type>Advance on Convertible Securities</type> # if its applicable it always be same value
<description>A form of bridge financing where investors provide capital upfront with the agreement to receive equity or another security type in a future round.</description>
</security>
<security>
<type>Partnership Interest</type> # if its applicable it always be same value
<description>An ownership stake in a partnership entity. Holders may receive distributions based on profits and have voting rights in the partnership's governance.</description>
</security>
<security>
<type>Membership Interest</type> # if its applicable it always be same value
<description>An ownership stake in an LLC (Limited Liability Company). Membership interests may include voting rights, profit distributions, and management participation.</description>
</security>
<security>
<type>SAFT</type>
<description>Similar to a SAFE but specifically used for token-based projects. Investors provide funds upfront in exchange for rights to receive future tokens once they are issued.</description>
</security>
<security>
<type>BOND</type>
<description>A fixed-income security representing a loan made by an investor to a borrower. The issuer agrees to repay the principal along with interest over a specified term.</description>
</security>
<security>
<type>PRIVATE INVESTMENT</type>
<description>An investment made directly into a private company. This can take various forms, including equity, debt, or hybrid instruments.</description>
</security>
<security>
<type>ORDINARY</type>
<description>Typically refers to ordinary shares, which are similar to common stock. Holders may have voting rights and share in the profits but are last in line for liquidation payouts.</description>
</security>
</securities>
## input
<input>
{security_type}
</input>
## Always output need to exact value and to be enclosed in <output> xml tags. Just give the output not need of any explanation
"""
ACCOUNT_TYPE_PROMPT = """
You are a Fund Expert in classifying account types. You are given account type and you need to look for exact match of the <input> into platform supported account type and if not it should match with other category which need to derive based on other category and you need to write into <output> xml tags.
# platform supported account types
<account-type>
Bank
Money market funds
Capital Call Receivable - LP
Portfolio Interest Receivable
Reimbursement Payable
Due to investment
Portfolio Interest Income
Due to Related Party
Due to Management Company
Investment - Unrealized Gain/Loss
Unrealized Gain/Loss
Portfolio Company Receivable
Realized Gain/Loss
Distribution Payable
Cash Distribution - LP
Due to General Partner
Misc Receivable
Temporarily Commingled Cash from Bank Receivable
Due from the Management Company
Receivable between Master Feeder
Receivable from Affiliate
Payable between Master Feeder
Payable to Affiliate
Due from Affiliates
Due to Affiliates
Accelaration Prepaids
Organization Costs Asset
Investment
Management Fee Payable
Admission Fee Payable
Admission Fees
Due to Limited Partner
Management Fees Prepaid
Due to Portfolio Company
Deferred Capital Calls - LP
Deferred Capital Calls - GP
Audit Accrual
Fund Administration Accrual
Tax Accrual
Syndication Costs - General
Contributed Capital - LP
Misc Income
Management Fees
Management fees offset
Audit Fees
Tax Prep Fees
Fund Administration Fees
Administration Fees
Administration Fees - Allocated from Master
Audit Fees - Allocated from Master
Bank Charges - Allocated from Master
Commission Expense
Commission Expense - Allocated from Master
FATCA Expenses
FATCA Expenses - Allocated from Master
FS Preparation
FS Preparation - Allocated From Master
Legal Expenses - Allocated from Master
Marketing Expenses - Allocated from Master
Offering & Organizational Cost
Offering & Organizational Cost - Allocated from Master
Performance Fees
Prepaid Offering & Organizational Cost
Travelling Expense
Travelling Expense - Allocated from Master
Administration Fees Payable
Bank Charges Payable
FATCA Payable
FS Prepartion Payable
Marketing Expenses Payable
Offering & Organizational Cost Payable
Performance Fees Payable
Travelling Expense Payable
Audit Fees Payable
Due to/from GP
Unrealized Gain (Loss) from Private Investment Companies
Unrealized Gain (Loss) from Private Investment Companies - Allocated from Master
Unrealized Gain (Loss) from Cryptocurrencies (Price) - Allocated from Master
Unrealized Gain (Loss) on Investment in Cryptocurrencies (Price)
Due from Limited Partner
Due/To from Offshore Feeder Fund
Due/To from Offshore Master Fund
Due/To from Onshore Feeder Fund
Receivable from pending settlement
Investment in Cryptocurrencies, (At Cost)
Investment in Fixed Income Bond
Investment in Master
Investment in Private Investment Companies
Investment In Safes, (At Cost)
Investment In SAFT, (At Cost)
Change in Unrealized Gain (Loss) from Private Investment Companies
Change in Unrealized Gain (Loss) on Investment in Cryptocurrencies (Price)
Change in Unrealized Gain (Loss) on Investment in Cryptocurrencies (Price) - Allocated from Master
Income from Master
Staking Income
Staking Income - Allocated from Master
Interest Income - Allocated from Master
Realized Gain (Loss) on Investment in Cryptocurrencies (Price)
Realized Gain (Loss) on Investment in Cryptocurrencies (Price) - Allocated from Master
Legal Fees
Legal Fees Accrual
Interest Payable
Loan Payable
Organization Costs Expense
Filing Fees
Office supplies
Subscription Fee - LP
Organization Costs Amortization
Distribution Payable - GP
Tax Preparation Fees Accrual
Escrow from portfolio investments sold
Cash Distribution - GP
Cash Carry Distributions
Carried Interest Distribution - GP
Bank Charges
Bank Interest Income
Consulting
Organization Expenses
Due Diligence
Interest Expense
Deal Fees
Due from General Partner
Dues and Subscriptions
Conference and Trade Shows
Website Design
Syndication Costs - Placement Agent Fees
Software
Reimbursement of Partnership Expenses
Contributed Capital - GP
Contributed capital - cashless offset - GP
Capital Call Receivable - GP
Expense Payable
Marketing Expenses
Travel
Meal
Acquisition Fee
Interest Income
Late Interest Income
Dividend Income
</account-type>
## if you are not able to find a exact math in the above <account-type> xml list then you need to pick the applicable value from the <other-type> xml data.
<other-type>
Due to Other
Other Prepaids
Other assets
Other Accrual
Other Professional Fees
Other Fees
Due from Related Parties
Other Interest Income
Misc Income
</other-type>
## input account type
<input>
{account_type}
<input>
Think step by step when deciding the the output and the output should be a exact match not close match with <account-types> and be a close match with <other-types>.
"""

373
task/utils/token_tracker.py Normal file
View File

@ -0,0 +1,373 @@
"""
Token usage tracker for AI file processing.
Records per-model input/output token counts during a processing session and
logs a record per model when the session ends. (Posting usage to the backend
``{BASE_URL}/api/v1/compliance/token`` is disabled — usage is logged only.)
Design — avoid double counting
------------------------------
LLM calls in this repo flow through several wrappers (GeminiBedrockClient,
think_tool, RLM engine). If every layer recorded usage, each API call would be
counted multiple times (the 5x bug we hit before).
Rule: record at the **terminal caller** — the function that issues a single
network call to the model. Shim layers (e.g. ``GeminiBedrockClient.invoke_model``
which only repackages a response) must NOT record.
Usage
-----
from utils.token_tracker import token_session, record_usage
with token_session(task_name="doc.pdf", entity_id=eid,
reference_id=event_id):
# any LLM call inside here will feed record_usage(model, in, out)
...
Thread-safety
-------------
Uses ``contextvars.ContextVar`` so each request/thread gets an isolated
session. The SQS processor's ThreadPoolExecutor calls ``handler()`` per
thread; each handler opens its own ``token_session``.
"""
from __future__ import annotations
import contextlib
import contextvars
import json
import logging
import os
import threading
from collections import defaultdict
from datetime import datetime, timezone
from typing import Optional
import requests
logger = logging.getLogger(__name__)
# USD per token. Source: Anthropic/Google published list prices.
_M = 1_000_000
# (input_price_per_token, output_price_per_token)
# Fund-upload flow standardizes on Claude Sonnet 4.6 — Bedrock and Vertex
# both use the same list price ($3 in / $15 out per 1M). Gemini entries are
# kept because image_template_analysis (document-studio flow) uses Gemini.
PRICING: dict[str, tuple[float, float]] = {
"claude-sonnet-4-6": (3.0 / _M, 15.0 / _M),
"anthropic.claude-sonnet-4-6": (3.0 / _M, 15.0 / _M),
"us.anthropic.claude-sonnet-4-6": (3.0 / _M, 15.0 / _M),
# Gemini 3 Pro family (image processor, separate flow from fund uploads)
"gemini-3-pro-preview": (2.0 / _M, 12.0 / _M),
"gemini-3-pro-image-preview": (2.0 / _M, 12.0 / _M),
# Gemini 2.5 Pro — default for LLM_BACKEND=gemini. List price for ≤ 200k
# prompt tokens (per https://ai.google.dev/gemini-api/docs/pricing —
# tiered: $1.25/$10 ≤200k, $2.50/$15 >200k incl. thinking). We bill the
# lower tier; ops can override via TOKEN_PRICING_OVERRIDES for >200k flows.
"gemini-2.5-pro": (1.25 / _M, 10.0 / _M),
}
# Fallback for unrecognized models — Sonnet 4.6 list price. This is a safety
# net only; unknown models log a WARNING so Ops can add them to PRICING.
_DEFAULT_PRICE = (3.0 / _M, 15.0 / _M)
_COMPLIANCE_PATH = "/api/v1/compliance/token"
# Ops can override pricing without a deploy by setting TOKEN_PRICING_OVERRIDES
# to a JSON object of {"model_id": [input_per_token, output_per_token]} or
# {"model_id": {"input_per_M": 3.0, "output_per_M": 15.0}}.
def _load_price_overrides() -> dict[str, tuple[float, float]]:
raw = os.getenv("TOKEN_PRICING_OVERRIDES", "").strip()
if not raw:
return {}
try:
data = json.loads(raw)
except Exception as e:
logger.warning("TOKEN_PRICING_OVERRIDES is not valid JSON: %s", e)
return {}
out: dict[str, tuple[float, float]] = {}
for model, val in data.items():
try:
if isinstance(val, (list, tuple)) and len(val) == 2:
out[model] = (float(val[0]), float(val[1]))
elif isinstance(val, dict):
pin = float(val.get("input_per_M", 0)) / _M
pout = float(val.get("output_per_M", 0)) / _M
out[model] = (pin, pout)
except Exception as e:
logger.warning("TOKEN_PRICING_OVERRIDES bad entry for %r: %s", model, e)
return out
PRICING.update(_load_price_overrides())
_unknown_models_warned: set[str] = set()
def _price_for(model: str) -> tuple[float, float]:
if not model:
return _DEFAULT_PRICE
if model in PRICING:
return PRICING[model]
# Prefix match: "us.anthropic.claude-sonnet-4-6:v1" etc.
for key, price in PRICING.items():
if model.startswith(key):
return price
if model not in _unknown_models_warned:
_unknown_models_warned.add(model)
logger.warning(
"token_tracker: no PRICING entry for %r — using default $3/$15 per 1M. "
"Add to PRICING or set TOKEN_PRICING_OVERRIDES to correct billing.",
model,
)
return _DEFAULT_PRICE
# Status values the backend AiUsageLog schema accepts. Anything else is
# rejected with a ValidatorError (served as a 500, so it would retry to
# exhaustion and then drop the record).
_BACKEND_STATUS_ENUM = {"in_progress", "completed", "failed"}
class _Session:
def __init__(
self,
task_name: str,
type_: str,
entity_id: str,
reference_id: Optional[str] = None,
reference_model: Optional[str] = None,
meta: Optional[dict] = None,
role_id: Optional[str] = None,
):
self.task_name = task_name
self.type = type_
self.entity_id = entity_id
self.reference_id = reference_id
self.reference_model = reference_model
self.meta = meta
self.role_id = role_id
self.started_at = datetime.now(timezone.utc).isoformat()
self.ended_at: Optional[str] = None
self.status = "in_progress"
self._lock = threading.Lock()
# {model: {"input": int, "output": int}}
self._usage: dict[str, dict[str, int]] = defaultdict(
lambda: {"input": 0, "output": 0}
)
def record(self, model: str, input_tokens: int, output_tokens: int) -> None:
if not (input_tokens or output_tokens):
return
model_key = model or "unknown"
with self._lock:
u = self._usage[model_key]
u["input"] += int(input_tokens or 0)
u["output"] += int(output_tokens or 0)
def set_status(self, status: str) -> None:
self.status = status
def set_meta(self, meta: Optional[dict]) -> None:
self.meta = meta
def to_records(self) -> list[dict]:
# The backend AiUsageLog schema only accepts _BACKEND_STATUS_ENUM.
# Custom statuses set via set_status (e.g. "duplicate",
# "classified_other") 500 with a ValidatorError on EVERY retry, then
# drop the record. Post a valid status and keep the semantic one in
# meta.statusDetail.
status = self.status
meta = self.meta
if status not in _BACKEND_STATUS_ENUM:
meta = {**(meta or {}), "statusDetail": status}
status = "completed"
records: list[dict] = []
with self._lock:
items = list(self._usage.items())
for model, u in items:
p_in, p_out = _price_for(model)
price = u["input"] * p_in + u["output"] * p_out
records.append({
"taskName": self.task_name,
"type": self.type,
"inputTokens": u["input"],
"outputTokens": u["output"],
"price": round(price, 6),
"startedAt": self.started_at,
"endedAt": self.ended_at,
"model": model,
"status": status,
"referenceId": self.reference_id,
"referenceModel": self.reference_model,
"meta": meta,
"entityId": self.entity_id,
})
if not records:
# Post a zero record so the backend still sees the session.
records.append({
"taskName": self.task_name,
"type": self.type,
"inputTokens": 0,
"outputTokens": 0,
"price": 0,
"startedAt": self.started_at,
"endedAt": self.ended_at,
"model": None,
"status": status,
"referenceId": self.reference_id,
"referenceModel": self.reference_model,
"meta": meta,
"entityId": self.entity_id,
})
return records
_current: contextvars.ContextVar[Optional[_Session]] = contextvars.ContextVar(
"_token_tracker_session", default=None,
)
def current_session() -> Optional[_Session]:
return _current.get()
_no_session_warned = False
def record_usage(model: str, input_tokens: int, output_tokens: int) -> None:
"""Terminal call sites invoke this after parsing a single API response.
If there is no active token_session we log a WARNING (once per process) so
Ops can catch regressions where an entry point forgot to open a session.
"""
sess = _current.get()
if sess is None:
if input_tokens or output_tokens:
global _no_session_warned
if not _no_session_warned:
_no_session_warned = True
logger.warning(
"record_usage(%s, in=%s, out=%s) called with NO active "
"token_session — usage NOT billed. Check that handler() / "
"process_single_document() opened a session around this path.",
model, input_tokens, output_tokens,
)
return
try:
sess.record(model, input_tokens, output_tokens)
except Exception as e: # never let tracking break the caller
logger.warning("record_usage failed: %s", e)
def _get_auth_token() -> Optional[str]:
# Reuse lambda_function.get_token() so we share the same auth flow.
try:
from lambda_function import get_token # type: ignore
return get_token()
except Exception:
pass
try:
from task.lambda_function import get_token # type: ignore
return get_token()
except Exception:
return None
_POST_MAX_ATTEMPTS = int(os.getenv("TOKEN_POST_MAX_ATTEMPTS", "5"))
_POST_TIMEOUT = float(os.getenv("TOKEN_POST_TIMEOUT", "15"))
# HTTP status codes where a retry is worthwhile (transient).
_RETRYABLE_STATUS = {408, 425, 429, 500, 502, 503, 504}
def _post_record(record: dict, role_id: Optional[str] = None) -> bool:
"""Log one compliance/token usage record.
Posting usage to the backend ({BASE_URL}/api/v1/compliance/token) has been
disabled for this service — token usage is recorded to the logs only, never
sent over the network. The ``role_id`` argument is kept for call-site
compatibility. Always returns True (the record was handled).
"""
logger.info(
"compliance/token usage model=%s in=%s out=%s price=%s role=%s "
"(log-only; not POSTed)",
record.get("model"), record.get("inputTokens"),
record.get("outputTokens"), record.get("price"), role_id,
)
return True
@contextlib.contextmanager
def token_session(
task_name: str,
entity_id: str,
type_: str = "ai_upload",
reference_id: Optional[str] = None,
reference_model: Optional[str] = None,
meta: Optional[dict] = None,
role_id: Optional[str] = None,
):
"""Open a tracking session. On exit, POST one record per model used.
Status is "completed" on clean exit and "failed" if an exception escapes
the block. Use ``session.set_status(...)`` inside the block to override
(e.g. to mark "duplicate" or "classified_other"). Custom statuses outside
the backend enum are POSTed as "completed" with the real value carried in
``meta.statusDetail`` — the backend rejects unknown enum values.
"""
sess = _Session(
task_name=task_name,
type_=type_,
entity_id=entity_id,
reference_id=reference_id,
reference_model=reference_model,
meta=meta,
role_id=role_id,
)
reset_token = _current.set(sess)
exc: Optional[BaseException] = None
try:
yield sess
except BaseException as e:
exc = e
raise
finally:
# An explicit set_status (e.g. "duplicate" before a control-flow
# raise) outranks the generic exception default.
if exc is not None and sess.status == "in_progress":
sess.status = "failed"
elif sess.status == "in_progress":
sess.status = "completed"
sess.ended_at = datetime.now(timezone.utc).isoformat()
# Emit an audit log line BEFORE POSTing. This gives Ops a
# guaranteed CloudWatch record even if every POST retry fails,
# so billing can be reconstructed from logs if needed.
records = sess.to_records()
try:
totals_in = sum(r.get("inputTokens", 0) for r in records)
totals_out = sum(r.get("outputTokens", 0) for r in records)
totals_price = sum(r.get("price", 0) for r in records)
logger.info(
"[TOKEN_AUDIT] task=%s entity=%s ref=%s type=%s status=%s "
"total_in=%d total_out=%d total_price=$%.6f models=%s "
"started=%s ended=%s",
sess.task_name, sess.entity_id, sess.reference_id,
sess.type, sess.status,
totals_in, totals_out, totals_price,
[r.get("model") for r in records],
sess.started_at, sess.ended_at,
)
except Exception as audit_err:
logger.warning("token audit log failed: %s", audit_err)
try:
for rec in records:
_post_record(rec, role_id=sess.role_id)
except Exception as post_err:
logger.error(
"token_session flush raised unexpectedly: %s — records were: %s",
post_err, json.dumps(records),
)
_current.reset(reset_token)

250
task/validatedUploadedFiles.py Normal file
View File

@ -0,0 +1,250 @@
import os
import boto3
from s3Ops import (
list_files_in_s3_folder
)
from utils.prompts import DOCUMENT_CLASSIFY_PROMPT
from backendAPIs import (
get_onboarding_status,
update_onboarding_status
)
bucket_name = os.getenv('S3_UPLOAD_BUCKET_NAME')
def document_classifier(record_details):
FINAL_DOCUMENT_CLASSIFY_PROMPT = DOCUMENT_CLASSIFY_PROMPT.format(document=record_details)
json_string = llm_bedrock.invoke(FINAL_DOCUMENT_CLASSIFY_PROMPT).content
print(json_string)
pattern = r'<output>(.*?)</output>'
match = re.search(pattern, json_string, re.DOTALL)
if match:
return match.group(1).strip()
else:
return ""
def get_pdf_details(key):
# Create a temporary file for the PDF
with tempfile.NamedTemporaryFile(delete=False, suffix='.pdf', dir='/tmp') as temp_file:
local_file_path = temp_file.name
# Download the file from S3
s3_client.download_file(bucket_name, key, local_file_path)
with pdfplumber.open(local_file_path) as pdf:
list_pages = []
for page in pdf.pages:
text = page.extract_text()
if text:
list_pages.append(text.strip())
pdf_string = "".join(list_pages)
if len(pdf_string.strip()) > 0:
# Clean up the temporary PDF file
os.remove(local_file_path)
return pdf_string
else: # start image processing
print("Starting image processing")
# Create a temporary directory for storing images
with tempfile.TemporaryDirectory(dir='/tmp') as temp_dir:
pdf_images_list = pdf_to_images(local_file_path, temp_dir)
result = read_images(pdf_images_list)
# Clean up the temporary PDF file
os.remove(local_file_path)
return result
def process_single_pdf(pdf_file):
pdf_content = get_pdf_details(key)
classification = document_classifier(pdf_content)
return pdf_file, classification
def pdf_classifier(files_list):
categories = {
"LPA(Limited Partner Agreement)": [],
"Capital Statements": [],
"Sub Docs(Subscription Document)": [],
"Side Letter": [],
"Capital Call Notices": [],
"Distribution Notices": [],
"Fund Reports": [],
"K1s": [],
"Other": []
}
with ThreadPoolExecutor(max_workers=10) as executor:
future_to_pdf = {executor.submit(process_single_pdf, pdf_file): pdf_file for pdf_file in pdf_files}
for future in as_completed(future_to_pdf):
pdf_file = future_to_pdf[future]
try:
pdf_file, classification = future.result()
# Determine which category the PDF belongs to
found_category = False
for category in categories.keys():
if category.lower() in classification.lower():
categories[category].append(pdf_file)
found_category = True
break
if not found_category:
categories["Other"].append(pdf_file)
except Exception as exc:
print(f'{pdf_file} generated an exception: {exc}')
categories["Other"].append(pdf_file)
return categories
def validate_input_files(file_list):
"""
Function to find specific files in a list based on naming patterns.
Args:
file_list (str): List of files to validate.
Returns:
dict: Dictionary containing file names that match the required patterns.
str: Error message if any required file is missing.
"""
# Patterns to look for
required_files = {
# "lpa": "_LP",
"partner": "_partner",
"financials": "_financials",
"bankTransactions": "_bank_transactions",
"journals": "_journals",
"fund_performance": "_fund-performance"
}
# Dictionary to hold found file names
found_files = {
# "lpa": "",
"partner": None,
"financials": None,
"bankTransactions": None,
"journals": None,
"fund_performance": None
}
pdf_files = []
try:
if file_list:
# Loop through files in the list
for file_name in file_list:
if required_files['partner'] in file_name:
found_files['partner'] = file_name
elif required_files['financials'] in file_name:
found_files['financials'] = file_name
elif required_files['bankTransactions'] in file_name:
found_files['bankTransactions'] = file_name
elif required_files['journals'] in file_name:
found_files['journals'] = file_name
elif required_files['fund_performance'] in file_name:
found_files['fund_performance'] = file_name
elif file_name.lower().endswith('.pdf'):
pdf_files.append(file_name)
else:
return(f"Error: No files found in folder.")
# if len(pdf_files) > 0:
# classified_pdf_files = pdf_classifier(pdf_files)
#
except FileNotFoundError:
return f"Error: Unable to find file.",""
# Check if all required files are found
missing_files = [key for key, value in found_files.items() if value is None]
if missing_files:
return f"Error: Missing required files - {', '.join(missing_files)}",""
# return found_files,classified_pdf_files
return found_files , pdf_files
def pretty_print_files(files_dict):
"""
Pretty print the dictionary containing file names.
Args:
files_dict (dict): Dictionary where the keys are the file categories, and the values are the file names.
"""
file_message = ""
for key, value in files_dict.items():
file_message += f"- {key.replace('_', ' ').capitalize()}: {value} \n"
return file_message
def run_file_validation():
bucket_name = os.getenv("S3_UPLOAD_BUCKET_NAME")
step_number = 1
# Array index starts from 0
step_number -= 1
print(f"Getting files from bucket {bucket_name}")
success_message = ""
error_message = ""
status = ""
try:
onboarding_id = os.getenv('ONBOARDING_ID')
print(f"Get onBoardingStatus for {onboarding_id}")
response = get_onboarding_status(onboarding_id)
# Get current onboarding status values.
onboarding_status = response['data']['data']
success_message = onboarding_status['steps'][step_number]['summary']
error_message = onboarding_status['steps'][step_number]['errorMessage']
status = onboarding_status['steps'][step_number]['status']
folder_path = onboarding_status['path']
print(f"Pulling files list from {bucket_name}/{folder_path}")
file_list = list_files_in_s3_folder(bucket_name, folder_path)
print("S3 files:", file_list)
found_files ,classified_files = validate_input_files(file_list)
print("Files found:", found_files)
if isinstance(found_files, dict):
success_message = pretty_print_files(found_files)
status = "COMPLETE"
#Validation failed
else:
error_message = found_files
status = "FAILED"
success_message = ""
response = update_onboarding_status(step_number, status, error_message, success_message)
if 'error' in response:
print(f"Failed to fetch onboarding status: Error: {response['error']}")
print(f"Status Code: {response['status_code']}")
return False , False
return found_files , classified_files
except Exception as e:
return False , False

52
task/zfunctions-prod.yaml Normal file
View File

@ -0,0 +1,52 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: prod-zfunctions-service
labels:
app: prod-zfunctions
spec:
replicas: 1
selector:
matchLabels:
app: prod-zfunctions
template:
metadata:
labels:
app: prod-zfunctions
spec:
serviceAccountName: ziveai-sa
containers:
- name: zfunctions-dev
image: <IMAGE>
ports:
- containerPort: 8080
envFrom:
- configMapRef:
name: prod-zfunctions-configmap
resources:
requests:
memory: "2Gi"
cpu: "1"
limits:
memory: "4Gi"
cpu: "2"
---
apiVersion: v1
kind: Service
metadata:
name: prod-zfunctions-service
labels:
app: prod-zfunctions
spec:
internalTrafficPolicy: Cluster
ipFamilies:
- IPv4
ipFamilyPolicy: SingleStack
selector:
app: prod-zfunctions
ports:
- protocol: TCP
port: 8080
targetPort: 8080
sessionAffinity: None
type: ClusterIP

52
task/zfunctions.yaml Normal file
View File

@ -0,0 +1,52 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: dev-zfunctions-service
labels:
app: dev-zfunctions
spec:
replicas: 1
selector:
matchLabels:
app: dev-zfunctions
template:
metadata:
labels:
app: dev-zfunctions
spec:
serviceAccountName: ziveai-sa
containers:
- name: zfunctions-dev
image: <IMAGE>
ports:
- containerPort: 8080
envFrom:
- configMapRef:
name: dev-zfunctions-configmap
resources:
requests:
memory: "2Gi"
cpu: "1"
limits:
memory: "4Gi"
cpu: "2"
---
apiVersion: v1
kind: Service
metadata:
name: dev-zfunctions-service
labels:
app: dev-zfunctions
spec:
internalTrafficPolicy: Cluster
ipFamilies:
- IPv4
ipFamilyPolicy: SingleStack
selector:
app: dev-zfunctions
ports:
- protocol: TCP
port: 8080
targetPort: 8080
sessionAffinity: None
type: ClusterIP

28
template.yml Normal file
View File

@ -0,0 +1,28 @@
AWSTemplateFormatVersion: '2010-09-09'
Transform: 'AWS::Serverless-2016-10-31'
Description: An AWS Lambda application that calls the Lambda API.
Resources:
function:
Type: AWS::Serverless::Function
Properties:
Handler: lambda_function.lambda_handler
Runtime: python3.8
CodeUri: function/.
Description: Call the AWS Lambda API
Timeout: 10
# Function's execution role
Policies:
- AWSLambdaBasicExecutionRole
- AWSLambda_ReadOnlyAccess
- AWSXrayWriteOnlyAccess
Tracing: Active
Layers:
- !Ref libs
libs:
Type: AWS::Serverless::LayerVersion
Properties:
LayerName: blank-python-lib
Description: Dependencies for the blank-python sample app.
ContentUri: package/.
CompatibleRuntimes:
- python3.8