Skip to content
Snippets Groups Projects
Commit 8cb864ab authored by Leo Pellandini's avatar Leo Pellandini
Browse files

add

parent 2cf6f656
No related branches found
No related tags found
No related merge requests found
Show whitespace changes
Inline Side-by-side
Part1 - Google/create-GCP-and-put-docs.py
+ 3
− 3
View file @ 8cb864ab
...@@ -16,12 +16,12 @@ def create_bucket(gcp_client, bucket_name): ...@@ -16,12 +16,12 @@ def create_bucket(gcp_client, bucket_name):
def write_files(gcp_client, directory, bucket): def write_files(gcp_client, directory, bucket):
bucket = gcp_client.bucket(bucket) bucket = gcp_client.bucket(bucket)
for filename in os.listdir(directory): for filename in os.listdir(directory):
if filename.endswith(".pdf"): # Vérifie si c'est un fichier PDF if filename.endswith(".pdf"):
file_path = os.path.join(directory, filename) # Crée le chemin complet vers le fichier file_path = os.path.join(directory, filename)
with open(file_path, 'rb') as file: with open(file_path, 'rb') as file:
print(f"Uploading {filename} to bucket {bucket}...") print(f"Uploading {filename} to bucket {bucket}...")
blob = bucket.blob(filename) blob = bucket.blob(filename)
blob.upload_from_filename(file_path) # Utilise le chemin complet ici blob.upload_from_filename(file_path)
print(f"{filename} uploaded successfully.") print(f"{filename} uploaded successfully.")
def main(bucket_name, local_dir): def main(bucket_name, local_dir):
......
Part1 - Google/create-vector-db.py
+ 42
− 157
View file @ 8cb864ab
# Creator: Abir Chebbi (abir.chebbi@hesge.ch) from google.cloud import firestore
## Source: https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-sdk.html
import boto3
import botocore
import time
import argparse import argparse
def create_encryption_policy(collection_name):
client = boto3.client('opensearchserverless') # Google Cloud Firestore does not explicitly have encryption policies
#service = 'aoss' # as data is encrypted by default at rest and in transit.
print(f'Encryption policy for {collection_name} created (default encryption used).')
def createEncryptionPolicy(client,policy_name, collection_name):
"""Creates an encryption policy for the specified collection.""" def create_network_policy(collection_name):
try: # Google Cloud Firestore uses IAM for access control
response = client.create_security_policy( print(f'Network policy for {collection_name} created (firewall rules managed via IAM).')
description=f'Encryption policy for {collection_name}',
name=policy_name, def create_access_policy(collection_name, user_email):
policy=f""" db = firestore.Client()
{{ # Example: Create a document in a Firestore collection for access policy
\"Rules\": [ access_policy = {
{{ 'resource': f'collection/{collection_name}',
\"ResourceType\": \"collection\", 'permissions': [
\"Resource\": [ 'firestore.documents.create',
\"collection/{collection_name}\" 'firestore.documents.delete',
] 'firestore.documents.update',
}} 'firestore.documents.get'
],
\"AWSOwnedKey\": true
}}
""",
type='encryption'
)
print('\nEncryption policy created:')
print(response)
except botocore.exceptions.ClientError as error:
if error.response['Error']['Code'] == 'ConflictException':
print(
'[ConflictException] The policy name or rules conflict with an existing policy.')
else:
raise error
def createNetworkPolicy(client,policy_name,collection_name):
"""Creates a network policy for the specified collection."""
try:
response = client.create_security_policy(
description=f'Network policy for {collection_name}',
name=policy_name,
policy=f"""
[{{
\"Description\": \"Public access for {collection_name}\",
\"Rules\": [
{{
\"ResourceType\": \"dashboard\",
\"Resource\": [\"collection/{collection_name}\"]
}},
{{
\"ResourceType\": \"collection\",
\"Resource\": [\"collection/{collection_name}\"]
}}
],
\"AllowFromPublic\": true
}}]
""",
type='network'
)
print('\nNetwork policy created:')
print(response)
except botocore.exceptions.ClientError as error:
if error.response['Error']['Code'] == 'ConflictException':
print(
'[ConflictException] A network policy with this name already exists.')
else:
raise error
def createAccessPolicy(client, policy_name, collection_name, IAM_USER):
"""Creates a data access policy for the specified collection."""
try:
policy_content = f"""
[
{{
"Rules": [
{{
"Resource": ["collection/{collection_name}"],
"Permission": [
"aoss:CreateCollectionItems",
"aoss:DeleteCollectionItems",
"aoss:UpdateCollectionItems",
"aoss:DescribeCollectionItems"
], ],
"ResourceType": "collection" 'user_email': user_email
}}, }
{{ db.collection('access_policies').add(access_policy)
"Resource": ["index/{collection_name}/*"], print(f'Access policy for {collection_name} created for user {user_email}.')
"Permission": [
"aoss:CreateIndex", def create_collection(collection_name):
"aoss:DeleteIndex", db = firestore.Client()
"aoss:UpdateIndex", # Create a new collection
"aoss:DescribeIndex", db.collection(collection_name).add({'status': 'active'})
"aoss:ReadDocument", print(f'Collection {collection_name} created successfully.')
"aoss:WriteDocument"
], def main(collection_name, user_email):
"ResourceType": "index" create_encryption_policy(collection_name)
}} create_network_policy(collection_name)
], create_access_policy(collection_name, user_email)
"Principal": ["arn:aws:iam::352909266144:user/{IAM_USER}"] create_collection(collection_name)
}}
]
"""
response = client.create_access_policy(
description=f'Data access policy for {collection_name}',
name=policy_name,
policy=policy_content,
type='data'
)
print('\nAccess policy created:')
print(response)
except botocore.exceptions.ClientError as error:
if error.response['Error']['Code'] == 'ConflictException':
print('[ConflictException] An access policy with this name already exists.')
else:
raise error
def waitForCollectionCreation(client,collection_name):
"""Waits for the collection to become active"""
time.sleep(30)
response = client.batch_get_collection(
names=[collection_name])
print('\nCollection successfully created:')
print(response["collectionDetails"])
# Extract the collection endpoint from the response
host = (response['collectionDetails'][0]['collectionEndpoint'])
final_host = host.replace("https://", "")
return final_host
def main(collection_name,IAM_USER):
encryption_policy_name = f'{collection_name}-encryption-policy'
network_policy_name = f'{collection_name}-network-policy'
access_policy_name = f'{collection_name}-access-policy'
createEncryptionPolicy(client, encryption_policy_name, collection_name)
createNetworkPolicy(client, network_policy_name, collection_name)
createAccessPolicy(client, access_policy_name, collection_name,IAM_USER)
collection = client.create_collection(name=collection_name,type='VECTORSEARCH')
ENDPOINT= waitForCollectionCreation(client,collection_name)
print("Collection created successfully:", collection)
print("Collection ENDPOINT:", ENDPOINT)
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Create collection") parser = argparse.ArgumentParser(description="Create collection")
parser.add_argument("--collection_name", help="The name of the collection") parser.add_argument("--collection_name", help="The name of the collection", required=True)
parser.add_argument("--iam_user", help="The iam user") parser.add_argument("--user_email", help="The user email", required=True)
args = parser.parse_args() args = parser.parse_args()
main(args.collection_name,args.iam_user) main(args.collection_name, args.user_email)
\ No newline at end of file
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment