Create Kafka cluster via Terraform

We recommend creating resources sequentially. If you create all resources at the same time, Terraform will take into account the dependencies between resources that you specified in the configuration file. If dependencies are not specified, resources will be created in parallel, which may lead to errors. For example, a resource required to create another resource may not be created yet.

Optional: configure providers.
Create a private network and subnet.
Create a cluster.
Create a user.
Create a topic.
Grant user access.

Configuration files

Example file for configuring providers

terraform {
  required_providers {
    selectel  = {
      source  = "selectel/selectel"
      version = "~> 8.6.0"
    }
    openstack = {
      source  = "terraform-provider-openstack/openstack"
      version = "3.4.0"
    }
  }
}

provider "selectel" {
  domain_name = "123456"
  username    = "user"
  password    = "password"
  auth_region = "ru-9"
  auth_url    = "https://cloud.api.selcloud.ru/identity/v3/"
}

resource "selectel_vpc_project_v2" "project_1" {
  name = "project"
}

resource "selectel_iam_serviceuser_v1" "serviceuser_1" {
  name         = "username"
  password     = "password"
  role {
    role_name  = "member"
    scope      = "project"
    project_id = selectel_vpc_project_v2.project_1.id 
  }
}

provider "openstack" {
  auth_url    = "https://cloud.api.selcloud.ru/identity/v3"
  domain_name = "123456"
  tenant_id   = selectel_vpc_project_v2.project_1.id
  user_name   = selectel_iam_serviceuser_v1.serviceuser_1.name
  password    = selectel_iam_serviceuser_v1.serviceuser_1.password
  region      = "ru-9"
}

Example file for creating a fixed configuration Kafka cluster

resource "openstack_networking_network_v2" "network_1" {
  name           = "private-network"
  admin_state_up = "true"
}

resource "openstack_networking_subnet_v2" "subnet_1" {
  network_id = openstack_networking_network_v2.network_1.id
  cidr       = "192.168.199.0/24"
}

data "selectel_dbaas_datastore_type_v1" "datastore_type_1" {
  project_id = selectel_vpc_project_v2.project_1.id
  region     = "ru-9"
  filter {
    engine  = "kafka"
    version = "3.5"
  }
}

data "selectel_dbaas_flavor_v1" "flavor_1" {
  project_id = selectel_vpc_project_v2.project_1.id
  region     = "ru-9"
  filter {
    datastore_type_id = data.selectel_dbaas_datastore_type_v1.datastore_type_1.datastore_types[0].id
    fl_size           = "standard"
    vcpus             = 4
    ram               = 16384
    disk              = 128
  }
}

resource "selectel_dbaas_kafka_datastore_v1" "datastore_1" {
  name       = "datastore-1"
  project_id = selectel_vpc_project_v2.project_1.id
  region     = "ru-9"
  type_id    = data.selectel_dbaas_datastore_type_v1.datastore_type_1.datastore_types[0].id
  subnet_id  = openstack_networking_subnet_v2.subnet_1.id
  node_count = 1
  flavor_id  = data.selectel_dbaas_flavor_v1.flavor_1.flavors[0].id
}

resource "selectel_dbaas_user_v1" "user_1" {
  project_id   = selectel_vpc_project_v2.project_1.id
  region       = "ru-9"
  datastore_id = selectel_dbaas_kafka_datastore_v1.datastore_1.id
  name         = "user"
  password     = "secret"
}

resource "selectel_dbaas_kafka_topic_v1" "topic_1" {
  project_id   = selectel_vpc_project_v2.project_1.id
  region       = "ru-9"
  datastore_id = selectel_dbaas_kafka_datastore_v1.datastore_1.id
  name         = "topic"
  partitions   = 1
}

resource "selectel_dbaas_kafka_acl_v1" "acl_1" {
  project_id   = selectel_vpc_project_v2.project_1.id
  region       = "ru-9"
  datastore_id = selectel_dbaas_kafka_datastore_v1.datastore_1.id
  pattern      = "topic"
  pattern_type = "prefixed"
  allow_read   = true
  allow_write  = true
  user_id      = selectel_dbaas_user_v1.user_1.id
}

Example file for creating a custom configuration Kafka cluster

resource "openstack_networking_network_v2" "network_1" {
  name           = "private-network"
  admin_state_up = "true"
}

resource "openstack_networking_subnet_v2" "subnet_1" {
  network_id = openstack_networking_network_v2.network_1.id
  cidr       = "192.168.199.0/24"
}

data "selectel_dbaas_datastore_type_v1" "datastore_type_1" {
  project_id = selectel_vpc_project_v2.project_1.id
  region     = "ru-9"
  filter {
    engine  = "kafka"
    version = "3.5"
  }
}

resource "selectel_dbaas_kafka_datastore_v1" "datastore_1" {
  name       = "datastore-1"
  project_id = selectel_vpc_project_v2.project_1.id
  region     = "ru-9"
  type_id    = data.selectel_dbaas_datastore_type_v1.datastore_type_1.datastore_types[0].id
  subnet_id  = selectel_vpc_subnet_v2.subnet.subnet_id
  node_count = 1
  flavor {
    vcpus = 2
    ram   = 8192
    disk  = 64
  }
}

resource "selectel_dbaas_user_v1" "user_1" {
  project_id   = selectel_vpc_project_v2.project_1.id
  region       = "ru-9"
  datastore_id = selectel_dbaas_kafka_datastore_v1.datastore_1.id
  name         = "user"
  password     = "secret"
}

resource "selectel_dbaas_kafka_topic_v1" "topic_1" {
  project_id   = selectel_vpc_project_v2.project_1.id
  region       = "ru-9"
  datastore_id = selectel_dbaas_kafka_datastore_v1.datastore_1.id
  name         = "topic"
  partitions   = 1
}

resource "selectel_dbaas_kafka_acl_v1" "acl_1" {
  project_id   = selectel_vpc_project_v2.project_1.id
  region       = "ru-9"
  datastore_id = selectel_dbaas_kafka_datastore_v1.datastore_1.id
  pattern      = "topic"
  pattern_type = "prefixed"
  allow_read   = true
  allow_write  = true
  user_id      = selectel_dbaas_user_v1.user_1.id
}

1. Optional: configure providers

If you have configured the Selectel and OpenStack providers, skip this step.

Make sure that in the control panel you have created a service user with the member roles in the Account scope and iam.admin.
Create a directory to store configuration files and a separate file with the .tf extension to configure providers.
Add the Selectel and OpenStack providers to the file for provider configuration:
```
terraform {
  required_providers {
    selectel  = {
      source  = "selectel/selectel"
      version = "~> 8.6.0"
    }
    openstack = {
      source  = "terraform-provider-openstack/openstack"
      version = "3.4.0"
    }
  }
}
```
Here version is provider versions. You can check the current version in the Selectel documentation (in the Terraform Registry and GitHub) and OpenStack (in the Terraform Registry and GitHub).

For more information about products, services, and features that can be managed using providers, see the Selectel and OpenStack Providers instructions.
Initialize the Selectel provider:
```
provider "selectel" {
  domain_name = "123456"
  username    = "user"
  password    = "password"
  auth_region = "ru-9"
  auth_url    = "https://cloud.api.selcloud.ru/identity/v3/"
}
```
Where:
- domain_name — Selectel account number. You can find it in the control panel in the upper-right corner;
- username — name of the service user with the member roles in the Account scope and iam.admin. You can find it in the control panel: in the top menu, click IAM → Service Users (the section is available only to the Account Owner and a user with the iam.admin role);
- password — password of the service user. You can view it when creating the user or change it to a new one;
- auth_region — pool for authorization in the ru-9 format; do not use pools in the SPB-2 format. The pool for authorization may differ from the pool where you create resources. You can check the list of available pools in the Product Availability by Location instructions.
Create a project:
```
resource "selectel_vpc_project_v2" "project_1" {
  name = "project"
}
```
See the detailed description of the selectel_vpc_project_v2 resource.
Create a service user to access the project and assign them the member role in the Project scope:
```
resource "selectel_iam_serviceuser_v1" "serviceuser_1" {
  name         = "username"
  password     = "password"
  role {
    role_name  = "member"
    scope      = "project"
    project_id = selectel_vpc_project_v2.project_1.id
  }
}
```
Where:
- username — username;
- password — user password. The password must be at least 20 characters long and include at least:
  - one uppercase and one lowercase Latin letter (A-Z, a-z);
  - one digit (0-9);
  - one special character from the ASCII Printable 7-Bit Special Characters list:
    !"#$%&'()*+,-./:;<=>?@[]^_{|}~;
- project_id — project ID. You can find it in the control panel: in the top menu, click IAM → Projects → in the row of the desired project, click .
See the detailed description of the selectel_iam_serviceuser_v1 resource.

Initialize the OpenStack provider:

provider "openstack" {
  auth_url    = "https://cloud.api.selcloud.ru/identity/v3"
  domain_name = "123456"
  tenant_id   = selectel_vpc_project_v2.project_1.id
  user_name   = selectel_iam_serviceuser_v1.serviceuser_1.name
  password    = selectel_iam_serviceuser_v1.serviceuser_1.password
  region      = "ru-9"
}

Where:

domain_name — Selectel account number. You can find it in the control panel in the upper-right corner;
region — pool, for example, ru-9. All resources will be created in this pool. You can check the list of available pools in the Product Availability by Location instructions.

If you create resources at the same time as configuring providers, add the depends_on argument for OpenStack resources. For example, for the openstack_networking_network_v2 resource:

resource "openstack_networking_network_v2" "network_1" {
  name           = "private-network"
  admin_state_up = "true"

  depends_on = [
    selectel_vpc_project_v2.project_1,
    selectel_iam_serviceuser_v1.serviceuser_1
  ]
}

Optional: if you want to use a mirror, create a separate Terraform CLI configuration file and add the following block to it:
```
provider_installation {
  network_mirror {
    url = "https://tf-proxy.selectel.ru/mirror/v1/"
    include = ["registry.terraform.io/*/*"]
  }
  direct {
    exclude = ["registry.terraform.io/*/*"]
  }
}
```
For more information about mirror settings, see the CLI Configuration File instructions in the HashiCorp documentation.
Open the CLI.
Initialize the Terraform configuration in the directory:
```
terraform init
```
Verify that the configuration files are syntactically correct:
```
terraform validate
```
Format the configuration files:
```
terraform fmt
```
Check which resources will be created:
```
terraform plan
```
Apply the changes and create the resources:
```
terraform apply
```
Confirm the creation — enter yes and press Enter. The created resources will appear in the control panel.
If there are not enough quotas to create resources, increase quotas.

2. Create a private network and subnet

resource "openstack_networking_network_v2" "network_1" {
  name           = "private-network"
  admin_state_up = "true"
}

resource "openstack_networking_subnet_v2" "subnet_1" {
  name       = "private-subnet"
  network_id = openstack_networking_network_v2.network_1.id
  cidr       = "192.168.199.0/24"
}

Here, cidr is the CIDR of the private subnet, for example, 192.168.199.0/24.

See the detailed resource description:

3. Create a cluster

Fixed configuration
Custom configuration

data "selectel_dbaas_datastore_type_v1" "datastore_type_1" {
  project_id = selectel_vpc_project_v2.project_1.id
  region     = "ru-9"
  filter {
    engine  = "kafka"
    version = "3.5"
  }
}

data "selectel_dbaas_flavor_v1" "flavor_1" {
  project_id = selectel_vpc_project_v2.project_1.id
  region     = "ru-9"
  filter {
    datastore_type_id = data.selectel_dbaas_datastore_type_v1.datastore_type_1.datastore_types[0].id
    fl_size           = "standard"
    vcpus             = 4
    ram               = 16384
    disk              = 128
  }
}

resource "selectel_dbaas_kafka_datastore_v1" "datastore_1" {
  name       = "datastore-1"
  project_id = selectel_vpc_project_v2.project_1.id
  region     = "ru-9"
  type_id    = data.selectel_dbaas_datastore_type_v1.datastore_type_1.datastore_types[0].id
  subnet_id  = openstack_networking_subnet_v2.subnet_1.id
  node_count = 1
  flavor_id  = data.selectel_dbaas_flavor_v1.flavor_1.flavors[0].id
}

Where:

region — pool, for example, ru-9. You can view the list of available pools in the instructions on Product availability by location;
filter — filter of Managed Databases types:
- engine — Managed Databases type;
- version — Managed Database version. You can view the list of available versions in the instructions on Versions and configurations;
nodes_count — number of nodes. The maximum number of nodes is 1;
filter — filter of fixed cluster configurations. You can view the list of available configurations in the instructions on Versions and configurations;
- fl_size — fixed configuration line. Available values: standard (for Standard, CPU, and Memory lines) and high_freq (for HighFreq line);
- vcpus — number of vCPUs;
- ram — RAM in MB;
- disk — disk size in GB.

See the detailed description of data sources and resources:

4. Create a user

resource "selectel_dbaas_user_v1" "user_1" {
  project_id   = selectel_vpc_project_v2.project_1.id
  region       = "ru-9"
  datastore_id = selectel_dbaas_kafka_datastore_v1.datastore_1.id
  name         = "user"
  password     = "secret"
}

Where:

region — pool where the cluster is located;
name — username;
password — user password.

See the detailed description of the selectel_dbaas_user_v1 resource.

5. Create a topic

resource "selectel_dbaas_kafka_topic_v1" "topic_1" {
  project_id   = selectel_vpc_project_v2.project_1.id
  region       = "ru-9"
  datastore_id = selectel_dbaas_kafka_datastore_v1.datastore_1.id
  name         = "topic"
  partitions   = 1
}

Where:

region — pool where the cluster is located;
partitions — number of partitions in the topic.

See the detailed description of the selectel_dbaas_kafka_topic_v1 resource.

6. Grant access to the user

resource "selectel_dbaas_kafka_acl_v1" "acl_1" {
  project_id   = selectel_vpc_project_v2.project_1.id
  region       = "ru-9"
  datastore_id = selectel_dbaas_kafka_datastore_v1.datastore_1.id
  pattern      = "topic"
  pattern_type = "prefixed"
  allow_read   = true
  allow_write  = true
  user_id      = selectel_dbaas_user_v1.user_1.id
}

Where:

region — pool where the cluster is located;
pattern — name or prefix of the topic to which access is granted. Skip this argument if pattern_type = all;
pattern_type — topic access type. Available values: prefixed, literal, all. If pattern_type = all, skip the pattern argument;
allow_read — grants the user the consumer role. The user receives and processes messages from the topic they have access to;
allow_write — grants the user the producer role. The user sends messages to the topic they have access to.

See the detailed description of the selectel_dbaas_kafka_acl_v1 resource.