diff --git a/.gitattributes b/.gitattributes
index 642db914..5b03d703 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -1 +1,2 @@
*.js linguist-detectable=false
+*.ts linguist-detectable=false
diff --git a/.gitignore b/.gitignore
index b4eb12db..d45de5ca 100644
--- a/.gitignore
+++ b/.gitignore
@@ -11,7 +11,5 @@ engine/bin/
/engine/configs/server.yml
/engine/configs/run_ci.yaml
/engine/configs/ci_checker.yml
-/packer/example.com.key
-cloudformation/output/*.yaml
engine/meta
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 8ea06b4d..da8a843d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -12,7 +12,6 @@ workflow:
include:
- local: 'engine/.gitlab-ci.yml'
- local: 'ui/.gitlab-ci.yml'
- - local: 'cloudformation/.gitlab-ci.yml'
empty-job:
stage: test
@@ -25,5 +24,4 @@ empty-job:
changes:
- engine/**/*
- ui/**/*
- - cloudformation/**/*
when: never
diff --git a/README.md b/README.md
index fb7f20e0..daa48447 100644
--- a/README.md
+++ b/README.md
@@ -57,7 +57,7 @@ For example, cloning a 1 TiB PostgreSQL database takes ~10 seconds. Dozens of in
Try it yourself right now:
- enter [the Database Lab Platform](https://console.postgres.ai/), join the "Demo" organization, and test cloning of ~1 TiB demo database, or
-- check out another demo setup, DLE CE: https://nik-tf-test.aws.postgres.ai:446/instance, use the token `demo` to enter (this setup has self-signed certificates, so ignore browser's complaints)
+- check out another demo setup, DLE CE: https://demo.aws.postgres.ai:446/instance, use the token `demo_token` to enter
## How it works
Thin cloning is fast because it uses [Copy-on-Write (CoW)](https://en.wikipedia.org/wiki/Copy-on-write#In_computer_storage). DLE supports two technologies to enable CoW and thin cloning: [ZFS](https://en.wikipedia.org/wiki/ZFS) (default) and [LVM](https://en.wikipedia.org/wiki/Logical_Volume_Manager_(Linux)).
diff --git a/assets/database-lab-dark-mode.png b/assets/database-lab-dark-mode.png
new file mode 100644
index 00000000..072f6fb3
Binary files /dev/null and b/assets/database-lab-dark-mode.png differ
diff --git a/assets/database-lab-dark-mode.svg b/assets/database-lab-dark-mode.svg
new file mode 100644
index 00000000..a867914c
--- /dev/null
+++ b/assets/database-lab-dark-mode.svg
@@ -0,0 +1,7 @@
+
diff --git a/assets/database-lab-light-mode.png b/assets/database-lab-light-mode.png
new file mode 100644
index 00000000..347745a2
Binary files /dev/null and b/assets/database-lab-light-mode.png differ
diff --git a/assets/database-lab-light-mode.svg b/assets/database-lab-light-mode.svg
new file mode 100644
index 00000000..5a3c1e88
--- /dev/null
+++ b/assets/database-lab-light-mode.svg
@@ -0,0 +1,7 @@
+
diff --git a/assets/dle-simple.png b/assets/dle-simple.png
new file mode 100644
index 00000000..9727a2d9
Binary files /dev/null and b/assets/dle-simple.png differ
diff --git a/assets/dle-simple.svg b/assets/dle-simple.svg
new file mode 100644
index 00000000..be858b03
--- /dev/null
+++ b/assets/dle-simple.svg
@@ -0,0 +1,6 @@
+
diff --git a/cloudformation/.gitlab-ci.yml b/cloudformation/.gitlab-ci.yml
deleted file mode 100644
index a2de1d71..00000000
--- a/cloudformation/.gitlab-ci.yml
+++ /dev/null
@@ -1,39 +0,0 @@
-build_aws_ami:
- rules:
- - if: $CI_COMMIT_TAG =~ /^cf\/[0-9.]+$/
- - if: $CI_COMMIT_TAG =~ /^cf\/[0-9.]+[\-_]*[a-zA-Z]+[a-zA-Z0-9.\-_]*[a-zA-Z0-9]+$/
- stage: build
- image: ubuntu:20.04
- before_script:
- - export CF_VERSION=$(echo ${CI_COMMIT_TAG#"cf/"})
- - export DLE_VERSION=$(echo ${CF_VERSION%_preview*})
- - export AWS_DEFAULT_REGION="us-east-1"
-
- script:
- - apt-get update -y && apt-get install -y curl jq unzip gnupg2 lsb-release software-properties-common
- - curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add -
- - apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main"
- - apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CC86BB64
- - add-apt-repository ppa:rmescandon/yq
- - apt-get update && apt-get install packer yq
-
- - cd packer
- - export PKR_VAR_dle_version="v${DLE_VERSION}"
- - export PKR_VAR_ami_name_prefix="DBLABserver"
- - packer build template.json.pkr.hcl
-
- - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
- - unzip awscliv2.zip
- - ./aws/install
-
- - cd ../cloudformation
- - export DLE_CF_TEMPLATE_FILE="output/dle_cf-${CF_VERSION}.yaml"
- - sed -e "s|\$DLE_DOCKER_IMAGE_VERSION|$DLE_VERSION|" dle_cf_template.yaml > ${DLE_CF_TEMPLATE_FILE}
- - bash getAMIs.sh
-
- - export PACKAGE_NAME="cloudformation"
- - 'curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file ${DLE_CF_TEMPLATE_FILE} "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/${PACKAGE_NAME}/${CF_VERSION}/dle_cf-${CF_VERSION}.yaml"'
-
- artifacts:
- paths:
- - cloudformation/output/*.yaml
diff --git a/cloudformation/DLE3-AWS_MP.xlsx b/cloudformation/DLE3-AWS_MP.xlsx
deleted file mode 100644
index bebb8f3b..00000000
Binary files a/cloudformation/DLE3-AWS_MP.xlsx and /dev/null differ
diff --git a/cloudformation/dle_cf_template.yaml b/cloudformation/dle_cf_template.yaml
deleted file mode 100644
index a33b4d4c..00000000
--- a/cloudformation/dle_cf_template.yaml
+++ /dev/null
@@ -1,552 +0,0 @@
-AWSTemplateFormatVersion: 2010-09-09
-Description: >-
- AWS CloudFormation template DLE_Instance_Host: Creates a single EC2 instance based on Database Lab Engine (DLE) AMI,
- configures DLE, launches the data retrieval process, eventually making it possible to create thin clones using DLE API, CLI, or UI.
- You will be billed for the AWS resources used if you create a stack from this template.
-Metadata:
- AWS::CloudFormation::Interface:
- ParameterGroups:
- -
- Label:
- default: "Amazon EC2 configuration"
- Parameters:
- - InstanceType
- - ZFSVolumeSize
- - SSHLocation
- - VPC
- - Subnet
- - KeyName
- - Label:
- default: "TLS certificate configuration"
- Parameters:
- - CertificateSubdomain
- - CertificateHostedZone
- - CertificateEmail
- -
- Label:
- default: "Source PostgreSQL parameters"
- Parameters:
- - SourceDatabaseSize
- -
- Label:
- default: "Advanced DLE configuration"
- Parameters:
- - DLEZFSDataSetsNumber
- ParameterLabels:
- KeyName:
- default: "Key pair"
- InstanceType:
- default: "Instance type"
- SSHLocation:
- default: "Connection source IP range"
- SourceDatabaseSize:
- default: "Total source database size in GiB"
- CertificateSubdomain:
- default: "Certificate subdomain"
- CertificateHostedZone:
- default: "Hosted zone"
- CertificateEmail:
- default: "Certificate email"
- DLEVerificationToken:
- default: "DLE verification token"
- DLEZFSDataSetsNumber:
- default: "Number of supported snapshots."
- VPC:
- default: "VPC security group"
- Subnet:
- default: "Subnet"
-Parameters:
- Subnet:
- Description: Subnet to attach EC2 machine.
- Type: AWS::EC2::Subnet::Id
- VPC:
- Description: VPC to attach EC2 machine.
- Type: AWS::EC2::VPC::Id
- ConstraintDescription: Can contain only ASCII characters and can not be empty.
- KeyName:
- Description: Name of an existing EC2 KeyPair to enable SSH access to the instance
- Type: 'AWS::EC2::KeyPair::KeyName'
- ConstraintDescription: Can contain only ASCII characters and can not be empty.
- InstanceType:
- Description: DLE EC2 instance type
- Type: String
- Default: m5.2xlarge
- AllowedValues:
- - r5.large
- - r5.xlarge
- - r5.2xlarge
- - r5.4xlarge
- - r5.8xlarge
- - r5.12xlarge
- - r5.16xlarge
- - r5.24xlarge
- - m5.large
- - m5.xlarge
- - m5.2xlarge
- - m5.4xlarge
- - m5.8xlarge
- - m5.12xlarge
- - m5.16xlarge
- - m5.24xlarge
- ConstraintDescription: must be a valid EC2 instance type.
- SSHLocation:
- Description: CIDR in format x.x.x.x/32 to allow one specific IP address access, 0.0.0.0/0 to allow all IP addresses access, or another CIDR range
- Type: String
- MinLength: '9'
- MaxLength: '18'
- AllowedPattern: '(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/(\d{1,2})'
- ConstraintDescription: Must be a valid IP CIDR range of the form x.x.x.x/x
- SourceDatabaseSize:
- Description: The size of the source databases used to calculate the size of EBS volume, in GiB
- Type: Number
- Default: 40
- CertificateSubdomain:
- Description: Subdomain to obtain a TLS certificate for (for example, dle). Leave it empty if you don't need SSL connection or don't have Route 53 hosted zone.
- Type: String
- CertificateHostedZone:
- Description: Hosted zone to obtain a TLS certificate for (for example, example.com). Leave it empty if you don't need SSL connection or don't have Route 53 hosted zone.
- Type: String
- CertificateEmail:
- Description: Email address for important account notifications about the issued TLS certificate. Leave it empty if you don't need SSL connection or don't have Route 53 hosted zone.
- Type: String
- AllowedPattern: '^$|[^\s@]+@[^\s@]+\.[^\s@]+'
- Default: ''
- ConstraintDescription: Must be a valid email of the form \'user@example.com\'
- DLEVerificationToken:
- Description: DLE verification token
- Type: String
- Default: "example-verification-token"
- MinLength: '9'
- MaxLength: '32'
- DLEZFSDataSetsNumber:
- Description: Number of database copies needed
- Type: Number
- Default: 2
- MinValue: 2
- MaxValue: 100
-Mappings:
- AWSInstanceType2Arch:
- r5.large:
- Arch: HVM64
- r5.xlarge:
- Arch: HVM64
- r5.2xlarge:
- Arch: HVM64
- r5.4xlarge:
- Arch: HVM64
- r5.8xlarge:
- Arch: HVM64
- r5.12xlarge:
- Arch: HVM64
- r5.16xlarge:
- Arch: HVM64
- r5.24xlarge:
- Arch: HVM64
- m5.large:
- Arch: HVM64
- m5.xlarge:
- Arch: HVM64
- m5.2xlarge:
- Arch: HVM64
- m5.4xlarge:
- Arch: HVM64
- m5.8xlarge:
- Arch: HVM64
- m5.12xlarge:
- Arch: HVM64
- m5.16xlarge:
- Arch: HVM64
- m5.24xlarge:
- Arch: HVM64
- AWSRegionArch2AMI:
- eu-north-1:
- HVM64: ami-0888261a1eacb636e
- ap-south-1:
- HVM64: ami-00539bfa7a6926e1b
- eu-west-3:
- HVM64: ami-038d1f1d1ef71112b
- eu-west-2:
- HVM64: ami-07c2bca027887871b
- eu-west-1:
- HVM64: ami-0e38f0f4f0acd49c2
- ap-northeast-3:
- HVM64: ami-01cd2976ef1688c25
- ap-northeast-2:
- HVM64: ami-049c608703690f99e
- ap-northeast-1:
- HVM64: ami-0cb59515cd67fdc93
- sa-east-1:
- HVM64: ami-0b3aeaa58412025de
- ca-central-1:
- HVM64: ami-075d0aae6fdd356b1
- ap-southeast-1:
- HVM64: ami-054e735ba76985f92
- ap-southeast-2:
- HVM64: ami-06558ef4fedcf3c2f
- eu-central-1:
- HVM64: ami-048a27a74e4c1239d
- us-east-1:
- HVM64: ami-0ed40b8023c788775
- us-east-2:
- HVM64: ami-0d6a0bd053962b66f
- us-west-1:
- HVM64: ami-0ef7453c037b624ec
- us-west-2:
- HVM64: ami-0bdf048f8e10f02eb
-Conditions:
- CreateSubDomain:
- !Not [!Equals [!Ref CertificateHostedZone, '']]
- NotCreateSubDomain:
- !Not [Condition: CreateSubDomain]
-
-Resources:
- IamSSMRole:
- Type: "AWS::IAM::Role"
- Properties:
- Description: "EC2 IAM role for SSM access"
- AssumeRolePolicyDocument:
- Version: "2012-10-17"
- Statement:
- - Effect: "Allow"
- Principal:
- Service:
- - "ec2.amazonaws.com"
- Action:
- - "sts:AssumeRole"
- ManagedPolicyArns:
- - "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"
- EC2SSMInstanceProfile:
- Type: "AWS::IAM::InstanceProfile"
- Properties:
- Roles:
- - Ref: "IamSSMRole"
- LambdaExecutionRole:
- Type: AWS::IAM::Role
- Properties:
- AssumeRolePolicyDocument:
- Version: '2012-10-17'
- Statement:
- - Effect: Allow
- Principal: {Service: [lambda.amazonaws.com]}
- Action: ['sts:AssumeRole']
- Path: "/"
- ManagedPolicyArns:
- - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole
- PowerFunction:
- Type: AWS::Lambda::Function
- Properties:
- Handler: index.handler
- Role: !GetAtt LambdaExecutionRole.Arn
- Code:
- ZipFile: !Sub |
- var response = require('cfn-response');
- exports.handler = function(event, context) {
- var result = parseInt(event.ResourceProperties.Op1)*(parseInt(event.ResourceProperties.Op2)+2);
- response.send(event, context, response.SUCCESS, {Value: result});
- };
- Runtime: nodejs14.x
- SizeCalculate:
- Type: Custom::Power
- Properties:
- ServiceToken: !GetAtt PowerFunction.Arn
- Op1: !Ref SourceDatabaseSize
- Op2: !Ref DLEZFSDataSetsNumber
-
- ZFSVolume:
- Type: AWS::EC2::Volume
- DeletionPolicy: Snapshot
- Properties:
- Encrypted: True
- AvailabilityZone: !GetAtt DLEInstance.AvailabilityZone
- Size: !GetAtt SizeCalculate.Value
- Tags:
- -
- Key: Name
- Value: dle-zfs-volume
- VolumeType: gp2
-
- DLEInstance:
- Type: 'AWS::EC2::Instance'
- Properties:
- ImageId: !FindInMap
- - AWSRegionArch2AMI
- - !Ref 'AWS::Region'
- - !FindInMap
- - AWSInstanceType2Arch
- - !Ref InstanceType
- - Arch
- InstanceType: !Ref InstanceType
- IamInstanceProfile: !Ref EC2SSMInstanceProfile
- SecurityGroupIds: !If
- - CreateSubDomain
- - - !GetAtt DLESecurityGroup.GroupId
- - !GetAtt DLEUISecurityGroup.GroupId
- - - !GetAtt DLESecurityGroup.GroupId
- KeyName: !Ref KeyName
- SubnetId: !Ref Subnet
- Tags:
- -
- Key: Name
- Value: "DLE Instance"
- UserData:
- Fn::Base64: !Sub | # No more Fn::Join needed
- #!/bin/bash
- set -ex
-
- sleep 30
-
- # This code tested and works on Ubuntu 20.04 (current base AMI)
- disk=$(lsblk -e7 --output PATH,NAME,FSTYPE --json | jq -r '.blockdevices[] | select(.children == null and .fstype == null) | .path ')
-
- sudo zpool create -f \
- -O compression=on \
- -O atime=off \
- -O recordsize=128k \
- -O logbias=throughput \
- -m /var/lib/dblab/dblab_pool \
- dblab_pool \
- $disk
-
- for i in {1..${DLEZFSDataSetsNumber}}; do
- sudo zfs create dblab_pool/dataset_$i
- done
-
- dle_config_path="/home/ubuntu/.dblab/engine/configs"
- dle_meta_path="/home/ubuntu/.dblab/engine/meta"
- postgres_conf_path="/home/ubuntu/.dblab/postgres_conf"
- dle_logs_path="/home/ubuntu/.dblab/engine/logs"
-
- yq e -i '
- .embeddedUI.host="" |
- .embeddedUI.dockerImage="registry.gitlab.com/postgres-ai/database-lab/ce-ui:$DLE_DOCKER_IMAGE_VERSION" |
- .server.verificationToken="${DLEVerificationToken}" |
- .poolManager.mountDir = "/var/lib/dblab/dblab_pool" |
- .retrieval.spec.logicalDump.options.dumpLocation="/var/lib/dblab/dblab_pool/dataset_1/dump/" |
- .retrieval.spec.logicalRestore.options.forceInit=true |
- .retrieval.spec.logicalRestore.options.dumpLocation="/var/lib/dblab/dblab_pool/dataset_1/dump/"
- ' $dle_config_path/server.yml
-
-
- touch $dle_meta_path/pending.retrieval
-
- sudo docker run \
- --detach \
- --name dblab_server \
- --label dblab_control \
- --privileged \
- --publish 2345:2345 \
- --volume /var/run/docker.sock:/var/run/docker.sock \
- --volume /var/lib/dblab:/var/lib/dblab/:rshared \
- --volume /var/lib/dblab/dblab_pool/dataset_1/dump/:/var/lib/dblab/dblab_pool/dataset_1/dump/:rshared \
- --volume $dle_config_path:/home/dblab/configs \
- --volume $dle_meta_path:/home/dblab/meta \
- --volume $dle_logs_path:/home/dblab/logs \
- --volume $postgres_conf_path:/home/dblab/standard/postgres/control \
- --volume /sys/kernel/debug:/sys/kernel/debug:rw \
- --volume /lib/modules:/lib/modules:ro \
- --volume /proc:/host_proc:ro \
- --env DLE_COMPUTING_INFRASTRUCTURE=AWS \
- --env DOCKER_API_VERSION=1.39 \
- --restart always \
- registry.gitlab.com/postgres-ai/database-lab/dblab-server:$DLE_DOCKER_IMAGE_VERSION
-
- if [ ! -z "${CertificateHostedZone}" ]; then
- export DOMAIN=${CertificateSubdomain}.${CertificateHostedZone}
- export USER_EMAIL=${CertificateEmail}
- export CERTIFICATE_EMAIL=${!USER_EMAIL:-'noreply@'$DOMAIN}
-
- sudo certbot certonly --standalone -d $DOMAIN -m $CERTIFICATE_EMAIL --agree-tos -n
- sudo cp /etc/letsencrypt/live/$DOMAIN/fullchain.pem /etc/envoy/certs/fullchain1.pem
- sudo cp /etc/letsencrypt/live/$DOMAIN/privkey.pem /etc/envoy/certs/privkey1.pem
-
- cat < /etc/letsencrypt/renewal-hooks/deploy/envoy.deploy
- #!/bin/bash
- umask 0177
- export DOMAIN=${CertificateSubdomain}.${CertificateHostedZone}
- export DATA_DIR=/etc/envoy/certs/
- cp /etc/letsencrypt/live/$DOMAIN/fullchain.pem $DATA_DIR/fullchain1.pem
- cp /etc/letsencrypt/live/$DOMAIN/privkey.pem $DATA_DIR/privkey1.pem
- EOF
- sudo chmod +x /etc/letsencrypt/renewal-hooks/deploy/envoy.deploy
-
- sudo systemctl enable envoy
- sudo systemctl start envoy
- fi
-
- # upgrade ssm agent version
- wget https://s3.us-east-1.amazonaws.com/amazon-ssm-us-east-1/amazon-ssm-agent/3.1.1575.0/amazon-ssm-agent-ubuntu-amd64.tar.gz
- tar -xf amazon-ssm-agent-ubuntu-amd64.tar.gz
- bash snap-install.sh
-
- # Initialize CLI configuration
- su ubuntu -c '/usr/local/bin/dblab init \
- --environment-id=test \
- --url=http://localhost:2345 \
- --token=${DLEVerificationToken} \
- --insecure'
-
- # Start Netdata
-
- touch $dle_config_path/netdata_DLE.conf
- echo "DLE_token=${DLEVerificationToken}"> $dle_config_path/netdata_DLE.conf
-
- sudo docker run -d \
- --name=netdata \
- --network host \
- --hostname=$(hostname) \
- -p 19999:19999 \
- -v netdataconfig:/etc/netdata \
- -v $dle_config_path/netdata_DLE.conf:/etc/netdata/charts.d/DLE.conf \
- -v netdatalib:/var/lib/netdata \
- -v netdatacache:/var/cache/netdata \
- -v /proc:/host/proc:ro \
- -v /sys:/host/sys:ro \
- -v /etc/os-release:/host/etc/os-release:ro \
- --restart unless-stopped \
- --cap-add SYS_PTRACE \
- --security-opt apparmor=unconfined \
- postgresai/netdata-for-dle:v1.34.1
-
- while ! echo "UI started" | nc localhost 2346; do sleep 10; done
- /opt/aws/bin/cfn-signal -e $? -d "DLE UI is available" -r "DLE Deploy Process Complete" '${WaitHandle}'
-
- WaitHandle:
- Type: AWS::CloudFormation::WaitConditionHandle
- WaitCondition:
- Type: AWS::CloudFormation::WaitCondition
- DependsOn: DLEInstance
- Properties:
- Handle: !Ref 'WaitHandle'
- Timeout: '600'
-
- MountPoint:
- Type: AWS::EC2::VolumeAttachment
- Properties:
- InstanceId: !Ref DLEInstance
- VolumeId: !Ref ZFSVolume
- Device: /dev/xvdh
-
- DLEElasticIP:
- Type: AWS::EC2::EIP
- Properties:
- Domain: vpc
- InstanceId: !Ref DLEInstance
-
- SubDomain:
- Type: AWS::Route53::RecordSet
- Condition: CreateSubDomain
- Properties:
- HostedZoneName: !Sub '${CertificateHostedZone}.'
- Comment: DNS name for DLE instance.
- Name: !Sub '${CertificateSubdomain}.${CertificateHostedZone}'
- Type: CNAME
- TTL: 60
- ResourceRecords:
- - !GetAtt DLEInstance.PublicDnsName
- DependsOn:
- - DLEInstance
- - DLEElasticIP
-
- DLESecurityGroup:
- Type: 'AWS::EC2::SecurityGroup'
- Properties:
- GroupDescription: Enable ssh access via port 22
- SecurityGroupIngress:
- - IpProtocol: tcp
- FromPort: 22
- ToPort: 22
- CidrIp: !Ref SSHLocation
- SecurityGroupEgress:
- - IpProtocol: -1
- CidrIp: '0.0.0.0/0'
- VpcId: !Ref VPC
-
- DLEUISecurityGroup:
- Type: 'AWS::EC2::SecurityGroup'
- Condition: CreateSubDomain
- Properties:
- GroupDescription: Enable ports to access DLE UI
- SecurityGroupIngress:
- - IpProtocol: tcp
- FromPort: 80
- ToPort: 80
- CidrIp: !Ref SSHLocation
-
- - IpProtocol: tcp
- FromPort: 443
- ToPort: 443
- CidrIp: !Ref SSHLocation
-
- - IpProtocol: tcp
- FromPort: 446
- ToPort: 446
- CidrIp: !Ref SSHLocation
- SecurityGroupEgress:
- - IpProtocol: -1
- CidrIp: '0.0.0.0/0'
- VpcId: !Ref VPC
-
-Outputs:
- 02VerificationToken:
- Description: 'DLE verification token'
- Value: !Ref DLEVerificationToken
-
- 08DLEInstance:
- Description: URL for newly created DLE instance
- Value: !Sub 'https://${CertificateSubdomain}.${CertificateHostedZone}'
- Condition: CreateSubDomain
-
- 01WebUIUrl:
- Description: UI URL with a domain for newly created DLE instance
- Value: !Sub 'https://${CertificateSubdomain}.${CertificateHostedZone}:446'
- Condition: CreateSubDomain
- 01WebUIUrl:
- Description: UI URL with a domain for newly created DLE instance
- Value: !Sub 'http://localhost:2346'
- Condition: NotCreateSubDomain
-
- 07EBSVolumeSize:
- Description: Size of provisioned EBS volume
- Value: !GetAtt SizeCalculate.Value
-
- 03DNSName:
- Description: Public DNS name
- Value: !GetAtt DLEInstance.PublicDnsName
-
- 06EC2SSH:
- Description: SSH connection to the EC2 instance with Database Lab Engine
- Value: !Sub
- - 'ssh ubuntu@${DNSName} -i YOUR_PRIVATE_KEY'
- - DNSName: !GetAtt DLEInstance.PublicDnsName
-
- 05DLETunnel:
- Description: Create an SSH-tunnel to Database Lab Engine
- Value: !Sub
- - 'ssh -N -L 2345:${DNSName}:2345 ubuntu@${DNSName} -i YOUR_PRIVATE_KEY'
- - DNSName: !GetAtt DLEInstance.PublicDnsName
-
- 00UITunnel:
- Description: Use SSH port forwarding to be able to access DLE UI
- Value: !Sub
- - 'ssh -N -L 2346:${DNSName}:2346 ubuntu@${DNSName} -i YOUR_PRIVATE_KEY'
- - DNSName: !GetAtt DLEInstance.PublicDnsName
-
- 07NetdataTunnel:
- Description: Use SSH port forwarding to be able to access monitoring (Netdata)
- Value: !Sub
- - 'ssh -N -L 19999:${DNSName}:19999 ubuntu@${DNSName} -i YOUR_PRIVATE_KEY'
- - DNSName: !GetAtt DLEInstance.PublicDnsName
- 08NetdataUrl:
- Description: Netdata URL with a domain for newly created DLE instance
- Value: !Sub 'https://${CertificateSubdomain}.${CertificateHostedZone}:19999'
- Condition: CreateSubDomain
- 08NetdataUrl:
- Description: Netdata URL with a domain for newly created DLE instance
- Value: !Sub 'http://localhost:19999'
- Condition: NotCreateSubDomain
-
- 04CloneTunnel:
- Description: Use SSH port forwarding to be able to access a database clone
- Value: !Sub
- - 'ssh -N -L CLONE_PORT:${DNSName}:CLONE_PORT ubuntu@${DNSName} -i YOUR_PRIVATE_KEY'
- - DNSName: !GetAtt DLEInstance.PublicDnsName
diff --git a/cloudformation/getAMIs.sh b/cloudformation/getAMIs.sh
deleted file mode 100755
index 1f8242a1..00000000
--- a/cloudformation/getAMIs.sh
+++ /dev/null
@@ -1,56 +0,0 @@
-#!/bin/bash
-# This script takes a a parameter which needs to be a name of an AWS AMI
-# The string will have to identify the AMI uniquely in all regions.
-# The script will generate an output which can be copied into json files of AWS CloudFormation
-#
-# The script uses the AWS command line tools.
-# The AWS command line tools have to have a default profile with the permission to
-# describe a region and to describe an image
-
-# The script can be run with normal OS user privileges.
-# The script is not supposed to modify anything.
-# There is no warranty. Please check the script upfront. You will use it on your own risk
-
-# String to be used when no AMI is available in region
-NOAMI="NOT_SUPPORTED"
-
-# Change your aws profile if needed here (example, " --profile default"):
-PROFILE=${DLE_AWS_PROFILE:-""}
-
-DLE_CF_TEMPLATE_FILE="${DLE_CF_TEMPLATE_FILE:-dle_cf.yaml}"
-
-# Check whether AWS CLI is installed and in search path
-if ! aws_loc="$(type -p "aws")" || [ -z "$aws_loc" ]; then
-echo "Error: Script requires AWS CLI . Install it and retry"
-exit 1
-fi
-
-# Check whether parameter has been provided
-if [ -z "$1" ]
-then
-NAME=DBLABserver*
-echo "No parameter provided."
-else
-NAME=$1
-fi
-echo "Will search for AMIs with name: ${NAME}"
-echo "---------------------------------------"
-
-# Clean AWSRegionArch2AMI list
-yq e -i 'del( .Mappings.AWSRegionArch2AMI )' ${DLE_CF_TEMPLATE_FILE}
-
-##NAME=DBLABserver*
-Regions=$(aws ec2 describe-regions --query "Regions[].{Name:RegionName}" --output text ${PROFILE})
-for i in $Regions; do
- AMI=`aws ec2 describe-images --owners 005923036815 --region $i --filters "Name=name,Values=${NAME}" --output json | jq -r '.Images | sort_by(.CreationDate) | last(.[]).ImageId'`
- if [ -z "$AMI" ]; then
- AMI=$NOAMI
- fi
-
- echo " "${i}: $'\n' " "HVM64: ${AMI}
-
- if [ "$AMI" != "$NOAMI" ]; then
- yq e -i ".Mappings.AWSRegionArch2AMI.\"${i}\" = {\"HVM64\": \"${AMI}\"}" ${DLE_CF_TEMPLATE_FILE}
- fi
-done
-
diff --git a/cloudformation/output/.gitkeep b/cloudformation/output/.gitkeep
deleted file mode 100644
index e69de29b..00000000
diff --git a/engine/.gitlab-ci.yml b/engine/.gitlab-ci.yml
index 8979b6cf..b445fa72 100644
--- a/engine/.gitlab-ci.yml
+++ b/engine/.gitlab-ci.yml
@@ -397,16 +397,15 @@ build-image-rc-client:
DOCKER_FILE: "Dockerfile.dblab-cli"
DOCKER_NAME: "postgresai/dblab"
-build-image-swagger-latest:
+build-image-swagger-release:
<<: *build_image_definition
<<: *only_tag_release
variables:
DOCKER_FILE: "Dockerfile.swagger-ui"
- DOCKER_NAME: "registry.gitlab.com/postgres-ai/database-lab/dblab-swagger-ui"
+ DOCKER_IMAGE_NAME: "registry.gitlab.com/postgres-ai/database-lab/dblab-swagger-ui"
before_script:
- export CLEAN_TAG=$(echo ${CI_COMMIT_TAG#"v"})
- - export LATEST_TAG=$(echo ${CLEAN_TAG%.*}-latest)
- - export TAGS="${DOCKER_NAME}:${LATEST_TAG}"
+ - export TAGS="${DOCKER_IMAGE_NAME}:${CLEAN_TAG}"
.bash-test: &bash_test
stage: integration-test
@@ -463,7 +462,7 @@ bash-test-14:
bash-test-15:
<<: *bash_test
variables:
- POSTGRES_VERSION: 15beta4
+ POSTGRES_VERSION: 15rc1
integration-test:
services:
@@ -483,3 +482,24 @@ integration-test:
script:
- cd engine
- make test-ci-integration
+
+## Deploy
+.deploy-definition: &deploy_definition
+ stage: deploy
+ image: dtzar/helm-kubectl:2.14.1
+ script:
+ - bash ./engine/scripts/do.sh subs_envs ./engine/deploy/swagger-ui.yaml /tmp/swagger-ui.yaml
+ - kubectl apply --filename /tmp/swagger-ui.yaml -n $NAMESPACE
+
+deploy-swagger-ui-tag-release:
+ <<: *only_tag_release
+ <<: *deploy_definition
+ environment:
+ name: production
+ variables:
+ ENV: production
+ NAMESPACE: production
+ DOCKER_IMAGE_NAME: "registry.gitlab.com/postgres-ai/database-lab/dblab-swagger-ui"
+ before_script:
+ - export CLEAN_TAG=$(echo ${CI_COMMIT_TAG#"v"})
+ - export TAG="${DOCKER_IMAGE_NAME}:${CLEAN_TAG}"
diff --git a/engine/api/swagger-spec/dblab_server_swagger.yaml b/engine/api/swagger-spec/dblab_server_swagger.yaml
index 65e28b6c..17a05994 100644
--- a/engine/api/swagger-spec/dblab_server_swagger.yaml
+++ b/engine/api/swagger-spec/dblab_server_swagger.yaml
@@ -1,8 +1,8 @@
swagger: "2.0"
info:
description: "This is a Database Lab Engine sample server."
- version: "2.5.0"
- title: "Database Lab"
+ version: "3.2.0"
+ title: "Database Lab Engine API"
contact:
email: "team@postgres.ai"
license:
@@ -461,6 +461,30 @@ paths:
schema:
$ref: "#/definitions/Error"
+ /instance/retrieval:
+ get:
+ tags:
+ - "instance"
+ summary: "Report state of retrieval subsystem"
+ description: ""
+ operationId: "instanceRetrieval"
+ produces:
+ - "application/json"
+ parameters:
+ - in: header
+ name: Verification-Token
+ type: string
+ required: true
+ responses:
+ 200:
+ description: "Successful operation"
+ schema:
+ $ref: "#/definitions/Retrieving"
+ 500:
+ description: "Internal server error"
+ schema:
+ $ref: "#/definitions/Error"
+
/healthz:
get:
tags:
@@ -592,6 +616,30 @@ paths:
schema:
$ref: "#/definitions/Error"
+ /admin/ws-auth:
+ post:
+ tags:
+ - "config"
+ summary: "Test source database"
+ description: ""
+ operationId: "testDBConnection"
+ consumes:
+ - "application/json"
+ parameters:
+ - in: header
+ name: Verification-Token
+ type: string
+ required: true
+ responses:
+ 200:
+ description: "Successful operation"
+ schema:
+ $ref: "#/definitions/WSToken"
+ 500:
+ description: "Internal server error"
+ schema:
+ $ref: "#/definitions/Error"
+
definitions:
Instance:
type: "object"
@@ -631,11 +679,15 @@ definitions:
properties:
version:
type: "string"
+ edition:
+ type: "string"
startedAt:
type: "string"
format: "date-time"
telemetry:
type: boolean
+ disableConfigModification:
+ type: boolean
PoolEntry:
type: "object"
@@ -710,6 +762,34 @@ definitions:
nextRefresh:
type: "string"
format: "date-time"
+ activity:
+ $ref: "#/definitions/Activity"
+
+ Activity:
+ type: "object"
+ properties:
+ source:
+ type: "array"
+ items:
+ $ref: "#/definitions/PGActivityEvent"
+ target:
+ type: "array"
+ items:
+ $ref: "#/definitions/PGActivityEvent"
+
+ PGActivityEvent:
+ type: "object"
+ properties:
+ user:
+ type: "string"
+ query:
+ type: "string"
+ duration:
+ type: "number"
+ waitEventType:
+ type: "string"
+ waitEvent:
+ type: "string"
Provisioner:
type: "object"
@@ -1020,6 +1100,17 @@ definitions:
type: "string"
password:
type: "string"
+ db_list:
+ type: "array"
+ items:
+ type: "string"
+
+ WSToken:
+ type: "object"
+ properties:
+ token:
+ type: "string"
+ description: "WebSocket token"
externalDocs:
diff --git a/engine/api/swagger-ui/index.css b/engine/api/swagger-ui/index.css
new file mode 100644
index 00000000..f2376fda
--- /dev/null
+++ b/engine/api/swagger-ui/index.css
@@ -0,0 +1,16 @@
+html {
+ box-sizing: border-box;
+ overflow: -moz-scrollbars-vertical;
+ overflow-y: scroll;
+}
+
+*,
+*:before,
+*:after {
+ box-sizing: inherit;
+}
+
+body {
+ margin: 0;
+ background: #fafafa;
+}
diff --git a/engine/api/swagger-ui/index.html b/engine/api/swagger-ui/index.html
index ebb550b3..84ae62d3 100644
--- a/engine/api/swagger-ui/index.html
+++ b/engine/api/swagger-ui/index.html
@@ -5,56 +5,15 @@
Swagger UI
+
-
-
-
+