Skip to main content

Create Kafka cluster via Terraform

Last update:

We recommend creating resources in order. If you create all resources at once, Terraform will account for dependencies between resources that you specified in the configuration file. If dependencies are not specified, resources will be created in parallel, which may lead to errors. For instance, a resource required for creating another resource might not have been created yet.


  1. Optional: configure providers.
  2. Create a private network and subnet.
  3. Create a cluster.
  4. Create a user.
  5. Create a topic.
  6. Grant access to the user

Configuration files

Example file for configuring providers
terraform {
  required_providers {
    selectel  = {
      source  = "selectel/selectel"
      version = "~> 6.0"
    }
    openstack = {
      source  = "terraform-provider-openstack/openstack"
      version = "2.1.0"
    }
  }
}

provider "selectel" {
  domain_name = "123456"
  username    = "user"
  password    = "password"
  auth_region = "ru-9"
  auth_url    = "https://cloud.api.selcloud.ru/identity/v3/"
}

resource "selectel_vpc_project_v2" "project_1" {
  name = "project"
}

resource "selectel_iam_serviceuser_v1" "serviceuser_1" {
  name         = "username"
  password     = "password"
  role {
    role_name  = "member"
    scope      = "project"
    project_id = selectel_vpc_project_v2.project_1.id 
  }
}

provider "openstack" {
  auth_url    = "https://cloud.api.selcloud.ru/identity/v3"
  domain_name = "123456"
  tenant_id   = selectel_vpc_project_v2.project_1.id
  user_name   = selectel_iam_serviceuser_v1.serviceuser_1.name
  password    = selectel_iam_serviceuser_v1.serviceuser_1.password
  region      = "ru-9"
}
Example file for creating a fixed configuration Kafka cluster
resource "openstack_networking_network_v2" "network_1" {
  name           = "private-network"
  admin_state_up = "true"
}

resource "openstack_networking_subnet_v2" "subnet_1" {
  network_id = openstack_networking_network_v2.network_1.id
  cidr       = "192.168.199.0/24"
}

data "selectel_dbaas_datastore_type_v1" "datastore_type_1" {
  project_id = selectel_vpc_project_v2.project_1.id
  region     = "ru-9"
  filter {
    engine  = "kafka"
    version = "3.5"
  }
}

data "selectel_dbaas_flavor_v1" "flavor_1" {
  project_id = selectel_vpc_project_v2.project_1.id
  region     = "ru-9"
  filter {
    datastore_type_id = data.selectel_dbaas_datastore_type_v1.datastore_type_1.datastore_types[0].id
    fl_size           = "standard"
    vcpus             = 4
    ram               = 16384
    disk              = 128
  }
}

resource "selectel_dbaas_kafka_datastore_v1" "datastore_1" {
  name       = "datastore-1"
  project_id = selectel_vpc_project_v2.project_1.id
  region     = "ru-9"
  type_id    = data.selectel_dbaas_datastore_type_v1.datastore_type_1.datastore_types[0].id
  subnet_id  = openstack_networking_subnet_v2.subnet_1.id
  node_count = 1
  flavor_id  = data.selectel_dbaas_flavor_v1.flavor_1.flavors[0].id
}

resource "selectel_dbaas_user_v1" "user_1" {
  project_id   = selectel_vpc_project_v2.project_1.id
  region       = "ru-9"
  datastore_id = selectel_dbaas_kafka_datastore_v1.datastore_1.id
  name         = "user"
  password     = "secret"
}

resource "selectel_dbaas_kafka_topic_v1" "topic_1" {
  project_id   = selectel_vpc_project_v2.project_1.id
  region       = "ru-9"
  datastore_id = selectel_dbaas_kafka_datastore_v1.datastore_1.id
  name         = "topic"
  partitions   = 1
}

resource "selectel_dbaas_kafka_acl_v1" "acl_1" {
  project_id   = selectel_vpc_project_v2.project_1.id
  region       = "ru-9"
  datastore_id = selectel_dbaas_kafka_datastore_v1.datastore_1.id
  pattern      = "topic"
  pattern_type = "prefixed"
  allow_read   = true
  allow_write  = true
  user_id      = selectel_dbaas_user_v1.user_1.id
}
Example file for creating a custom configuration Kafka cluster
resource "openstack_networking_network_v2" "network_1" {
  name           = "private-network"
  admin_state_up = "true"
}

resource "openstack_networking_subnet_v2" "subnet_1" {
  network_id = openstack_networking_network_v2.network_1.id
  cidr       = "192.168.199.0/24"
}

data "selectel_dbaas_datastore_type_v1" "datastore_type_1" {
  project_id = selectel_vpc_project_v2.project_1.id
  region     = "ru-9"
  filter {
    engine  = "kafka"
    version = "3.5"
  }
}

resource "selectel_dbaas_kafka_datastore_v1" "datastore_1" {
  name       = "datastore-1"
  project_id = selectel_vpc_project_v2.project_1.id
  region     = "ru-9"
  type_id    = data.selectel_dbaas_datastore_type_v1.datastore_type_1.datastore_types[0].id
  subnet_id  = selectel_vpc_subnet_v2.subnet.subnet_id
  node_count = 1
  flavor {
    vcpus = 2
    ram   = 8192
    disk  = 64
  }
}

resource "selectel_dbaas_user_v1" "user_1" {
  project_id   = selectel_vpc_project_v2.project_1.id
  region       = "ru-9"
  datastore_id = selectel_dbaas_kafka_datastore_v1.datastore_1.id
  name         = "user"
  password     = "secret"
}

resource "selectel_dbaas_kafka_topic_v1" "topic_1" {
  project_id   = selectel_vpc_project_v2.project_1.id
  region       = "ru-9"
  datastore_id = selectel_dbaas_kafka_datastore_v1.datastore_1.id
  name         = "topic"
  partitions   = 1
}

resource "selectel_dbaas_kafka_acl_v1" "acl_1" {
  project_id   = selectel_vpc_project_v2.project_1.id
  region       = "ru-9"
  datastore_id = selectel_dbaas_kafka_datastore_v1.datastore_1.id
  pattern      = "topic"
  pattern_type = "prefixed"
  allow_read   = true
  allow_write  = true
  user_id      = selectel_dbaas_user_v1.user_1.id
}

1. Optional: configure providers

If you have configured the Selectel and OpenStack providers, skip this step.

  1. Make sure that in the control panel you have created a service user with the member role in the Account access scope and iam.admin.

  2. Create a directory to store configuration files and a separate file with the .tf extension to configure providers.

  3. Add the Selectel and OpenStack providers to the file for provider configuration:

    terraform {
      required_providers {
        selectel  = {
          source  = "selectel/selectel"
          version = "~> 7.1.0"
        }
        openstack = {
          source  = "terraform-provider-openstack/openstack"
          version = "2.1.0"
        }
      }
    }

    Here version is the provider version. The current version can be found in the Selectel documentation (in Terraform Registry and GitHub) and OpenStack (in Terraform Registry and GitHub).

    For more information about products, services, and features that can be managed using providers, see the Selectel and OpenStack Providers guide.

  4. Initialize the Selectel provider:

    provider "selectel" {
      domain_name = "123456"
      username    = "user"
      password    = "password"
      auth_region = "ru-9"
      auth_url    = "https://cloud.api.selcloud.ru/identity/v3/"
    }

    Where:

    • domain_name — Selectel account number. You can find it in the control panel in the top-right corner;
    • username — name of the service user with the member role in the Account access scope and iam.admin. You can view it in the control panel: in the top menu, click IAM → Service Users section (the section is only available to the Account Owner and a user with the iam.admin role);
    • password — service user password. You can view it when creating the user or change it to a new one;
    • auth_region — pool for authorization, for example, ru-9. You can create resources in other pools. A list of available pools can be found in the Availability Matrix guide.

  5. Create a project:

    resource "selectel_vpc_project_v2" "project_1" {
      name = "project"
    }

    See a detailed description of the selectel_vpc_project_v2 resource.

  6. Create a service user to access the project and assign them the member role in the Project access scope:

    resource "selectel_iam_serviceuser_v1" "serviceuser_1" {
      name         = "username"
      password     = "password"
      role {
        role_name  = "member"
        scope      = "project"
        project_id = selectel_vpc_project_v2.project_1.id
      }
    }

    Where:

    • username — username;

    • password — user password. The password must be at least 20 characters long and include at least:

      • one uppercase and one lowercase Latin letter (A-Z, a-z);
      • one digit (0-9);
      • one special character from the ASCII Printable 7-Bit Special Characters list:
        !"#$%&'()*+,-./:;<=>?@[]^_{|}~;

    • project_id — project ID. You can find it in the control panel: in the top menu, click Products and select Cloud Servers → open the projects menu → in the row of the target project, click .

    See a detailed description of the selectel_iam_serviceuser_v1 resource.

  7. Initialize the OpenStack provider:

    provider "openstack" {
      auth_url    = "https://cloud.api.selcloud.ru/identity/v3"
      domain_name = "123456"
      tenant_id   = selectel_vpc_project_v2.project_1.id
      user_name   = selectel_iam_serviceuser_v1.serviceuser_1.name
      password    = selectel_iam_serviceuser_v1.serviceuser_1.password
      region      = "ru-9"
    }

    Where:

    • domain_name — Selectel account number. You can find it in the control panel in the top-right corner;
    • region — pool, for example, ru-9. All resources will be created in this pool. A list of available pools can be found in the Availability Matrix guide.

  8. If you are creating resources while configuring providers, add the depends_on argument for OpenStack resources. For example, for the openstack_networking_network_v2 resource:

    resource "openstack_networking_network_v2" "network_1" {
      name           = "private-network"
      admin_state_up = "true"

      depends_on = [
        selectel_vpc_project_v2.project_1,
        selectel_iam_serviceuser_v1.serviceuser_1
      ]
    }

  9. Optional: if you want to use a mirror, create a separate Terraform CLI configuration file and add the following block to it:

    provider_installation {
      network_mirror {
        url = "https://tf-proxy.selectel.ru/mirror/v1/"
        include = ["registry.terraform.io/*/*"]
      }
      direct {
        exclude = ["registry.terraform.io/*/*"]
      }
    }

    Read more about mirror settings in the CLI Configuration File guide in the HashiCorp documentation.

  10. Open the CLI.

  11. Initialize the Terraform configuration in the directory:

    terraform init

  12. Verify that the configuration files are syntactically correct:

    terraform validate

  13. Format the configuration files:

    terraform fmt

  14. Check which resources will be created:

    terraform plan

  15. Apply the changes and create the resources:

    terraform apply

  16. Confirm creation — enter yes and press Enter. The created resources will appear in the control panel.

  17. If quotas were insufficient to create the resources, increase the quotas.

2. Create a private network and subnet

resource "openstack_networking_network_v2" "network_1" {
  name           = "private-network"
  admin_state_up = "true"
}

resource "openstack_networking_subnet_v2" "subnet_1" {
  name       = "private-subnet"
  network_id = openstack_networking_network_v2.network_1.id
  cidr       = "192.168.199.0/24"
}

Here cidr is the private subnet CIDR, for example 192.168.199.0/24.

See the detailed resource description:

3. Create a cluster

data "selectel_dbaas_datastore_type_v1" "datastore_type_1" {
  project_id = selectel_vpc_project_v2.project_1.id
  region     = "ru-9"
  filter {
    engine  = "kafka"
    version = "3.5"
  }
}

data "selectel_dbaas_flavor_v1" "flavor_1" {
  project_id = selectel_vpc_project_v2.project_1.id
  region     = "ru-9"
  filter {
    datastore_type_id = data.selectel_dbaas_datastore_type_v1.datastore_type_1.datastore_types[0].id
    fl_size           = "standard"
    vcpus             = 4
    ram               = 16384
    disk              = 128
  }
}

resource "selectel_dbaas_kafka_datastore_v1" "datastore_1" {
  name       = "datastore-1"
  project_id = selectel_vpc_project_v2.project_1.id
  region     = "ru-9"
  type_id    = data.selectel_dbaas_datastore_type_v1.datastore_type_1.datastore_types[0].id
  subnet_id  = openstack_networking_subnet_v2.subnet_1.id
  node_count = 1
  flavor_id  = data.selectel_dbaas_flavor_v1.flavor_1.flavors[0].id
}

Where:

  • region — pool, e.g. ru-9. The list of available pools can be found in the Availability Matrix article;
  • filter — database type filter:
    • engine — database type;
    • version — cloud database version. The list of available versions can be found in the Versions and configurations article;
  • nodes_count — number of nodes. The maximum number of nodes is 1;
  • filter — fixed cluster configuration filter. The list of available configurations can be found in the Versions and configurations article;
    • fl_size — fixed configuration line. Available values are standard (for Standard, CPU, and Memory lines) and high_freq (for HighFreq line);
    • vcpus — number of vCPUs;
    • ram — amount of RAM in MB;
    • disk — disk size in GB.

See the detailed description of data sources and resources:

4. Create a user

resource "selectel_dbaas_user_v1" "user_1" {
  project_id   = selectel_vpc_project_v2.project_1.id
  region       = "ru-9"
  datastore_id = selectel_dbaas_kafka_datastore_v1.datastore_1.id
  name         = "user"
  password     = "secret"
}

Where:

  • region — pool where the cluster is located;
  • name — username;
  • password — user password.

See the detailed description of the selectel_dbaas_user_v1.

5. Create a topic

resource "selectel_dbaas_kafka_topic_v1" "topic_1" {
  project_id   = selectel_vpc_project_v2.project_1.id
  region       = "ru-9"
  datastore_id = selectel_dbaas_kafka_datastore_v1.datastore_1.id
  name         = "topic"
  partitions   = 1
}

Where:

  • region — pool where the cluster is located;
  • partitions — number of partitions in the topic.

See the detailed description of the selectel_dbaas_kafka_topic_v1.

6. Grant access to the user

resource "selectel_dbaas_kafka_acl_v1" "acl_1" {
  project_id   = selectel_vpc_project_v2.project_1.id
  region       = "ru-9"
  datastore_id = selectel_dbaas_kafka_datastore_v1.datastore_1.id
  pattern      = "topic"
  pattern_type = "prefixed"
  allow_read   = true
  allow_write  = true
  user_id      = selectel_dbaas_user_v1.user_1.id
}

Where:

  • region — pool where the cluster is located;
  • pattern — name or prefix of the topic to which access is granted. Omit this argument if pattern_type = all;
  • pattern_type — topic access type. Available values are prefixed, literal, all. If pattern_type = all, omit the pattern;
  • allow_read — grants the consumer role to the user. The user receives and processes messages from the topic they have access to;
  • allow_write — grants the producer role to the user. The user sends messages to the topic they have access to.

See the detailed description of the selectel_dbaas_kafka_acl_v1.