Skip to content
Snippets Groups Projects
Commit 8cb864ab authored by Leo Pellandini's avatar Leo Pellandini
Browse files

add

parent 2cf6f656
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Part1 - Google/create-GCP-and-put-docs.py
+ 3
− 3
View file @ 8cb864ab
......@@ -16,12 +16,12 @@ def create_bucket(gcp_client, bucket_name):
def write_files(gcp_client, directory, bucket):
bucket = gcp_client.bucket(bucket)
for filename in os.listdir(directory):
if filename.endswith(".pdf"): # Vérifie si c'est un fichier PDF
file_path = os.path.join(directory, filename) # Crée le chemin complet vers le fichier
if filename.endswith(".pdf"):
file_path = os.path.join(directory, filename)
with open(file_path, 'rb') as file:
print(f"Uploading {filename} to bucket {bucket}...")
blob = bucket.blob(filename)
blob.upload_from_filename(file_path) # Utilise le chemin complet ici
blob.upload_from_filename(file_path)
print(f"{filename} uploaded successfully.")
def main(bucket_name, local_dir):
......
Part1 - Google/create-vector-db.py
+ 42
− 157
View file @ 8cb864ab
# Creator: Abir Chebbi (abir.chebbi@hesge.ch)
## Source: https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-sdk.html
import boto3
import botocore
import time
from google.cloud import firestore
import argparse
client = boto3.client('opensearchserverless')
#service = 'aoss'
def createEncryptionPolicy(client,policy_name, collection_name):
"""Creates an encryption policy for the specified collection."""
try:
response = client.create_security_policy(
description=f'Encryption policy for {collection_name}',
name=policy_name,
policy=f"""
{{
\"Rules\": [
{{
\"ResourceType\": \"collection\",
\"Resource\": [
\"collection/{collection_name}\"
]
}}
],
\"AWSOwnedKey\": true
}}
""",
type='encryption'
)
print('\nEncryption policy created:')
print(response)
except botocore.exceptions.ClientError as error:
if error.response['Error']['Code'] == 'ConflictException':
print(
'[ConflictException] The policy name or rules conflict with an existing policy.')
else:
raise error
def createNetworkPolicy(client,policy_name,collection_name):
"""Creates a network policy for the specified collection."""
try:
response = client.create_security_policy(
description=f'Network policy for {collection_name}',
name=policy_name,
policy=f"""
[{{
\"Description\": \"Public access for {collection_name}\",
\"Rules\": [
{{
\"ResourceType\": \"dashboard\",
\"Resource\": [\"collection/{collection_name}\"]
}},
{{
\"ResourceType\": \"collection\",
\"Resource\": [\"collection/{collection_name}\"]
}}
],
\"AllowFromPublic\": true
}}]
""",
type='network'
)
print('\nNetwork policy created:')
print(response)
except botocore.exceptions.ClientError as error:
if error.response['Error']['Code'] == 'ConflictException':
print(
'[ConflictException] A network policy with this name already exists.')
else:
raise error
def createAccessPolicy(client, policy_name, collection_name, IAM_USER):
"""Creates a data access policy for the specified collection."""
try:
policy_content = f"""
[
{{
"Rules": [
{{
"Resource": ["collection/{collection_name}"],
"Permission": [
"aoss:CreateCollectionItems",
"aoss:DeleteCollectionItems",
"aoss:UpdateCollectionItems",
"aoss:DescribeCollectionItems"
],
"ResourceType": "collection"
}},
{{
"Resource": ["index/{collection_name}/*"],
"Permission": [
"aoss:CreateIndex",
"aoss:DeleteIndex",
"aoss:UpdateIndex",
"aoss:DescribeIndex",
"aoss:ReadDocument",
"aoss:WriteDocument"
],
"ResourceType": "index"
}}
],
"Principal": ["arn:aws:iam::352909266144:user/{IAM_USER}"]
}}
]
"""
response = client.create_access_policy(
description=f'Data access policy for {collection_name}',
name=policy_name,
policy=policy_content,
type='data'
)
print('\nAccess policy created:')
print(response)
except botocore.exceptions.ClientError as error:
if error.response['Error']['Code'] == 'ConflictException':
print('[ConflictException] An access policy with this name already exists.')
else:
raise error
def waitForCollectionCreation(client,collection_name):
"""Waits for the collection to become active"""
time.sleep(30)
response = client.batch_get_collection(
names=[collection_name])
print('\nCollection successfully created:')
print(response["collectionDetails"])
# Extract the collection endpoint from the response
host = (response['collectionDetails'][0]['collectionEndpoint'])
final_host = host.replace("https://", "")
return final_host
def main(collection_name,IAM_USER):
encryption_policy_name = f'{collection_name}-encryption-policy'
network_policy_name = f'{collection_name}-network-policy'
access_policy_name = f'{collection_name}-access-policy'
createEncryptionPolicy(client, encryption_policy_name, collection_name)
createNetworkPolicy(client, network_policy_name, collection_name)
createAccessPolicy(client, access_policy_name, collection_name,IAM_USER)
collection = client.create_collection(name=collection_name,type='VECTORSEARCH')
ENDPOINT= waitForCollectionCreation(client,collection_name)
print("Collection created successfully:", collection)
print("Collection ENDPOINT:", ENDPOINT)
if __name__== "__main__":
def create_encryption_policy(collection_name):
# Google Cloud Firestore does not explicitly have encryption policies
# as data is encrypted by default at rest and in transit.
print(f'Encryption policy for {collection_name} created (default encryption used).')
def create_network_policy(collection_name):
# Google Cloud Firestore uses IAM for access control
print(f'Network policy for {collection_name} created (firewall rules managed via IAM).')
def create_access_policy(collection_name, user_email):
db = firestore.Client()
# Example: Create a document in a Firestore collection for access policy
access_policy = {
'resource': f'collection/{collection_name}',
'permissions': [
'firestore.documents.create',
'firestore.documents.delete',
'firestore.documents.update',
'firestore.documents.get'
],
'user_email': user_email
}
db.collection('access_policies').add(access_policy)
print(f'Access policy for {collection_name} created for user {user_email}.')
def create_collection(collection_name):
db = firestore.Client()
# Create a new collection
db.collection(collection_name).add({'status': 'active'})
print(f'Collection {collection_name} created successfully.')
def main(collection_name, user_email):
create_encryption_policy(collection_name)
create_network_policy(collection_name)
create_access_policy(collection_name, user_email)
create_collection(collection_name)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Create collection")
parser.add_argument("--collection_name", help="The name of the collection")
parser.add_argument("--iam_user", help="The iam user")
parser.add_argument("--collection_name", help="The name of the collection", required=True)
parser.add_argument("--user_email", help="The user email", required=True)
args = parser.parse_args()
main(args.collection_name,args.iam_user)
main(args.collection_name, args.user_email)
\ No newline at end of file
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment