使用 GitLab CI/CD 和 Terraform 实现 Lambda 以进行 SFTP 集成、Go 中的 S Databricks

package main

import (
 "fmt"
 "log"
 "os"
 "path/filepath"

 "github.com/pkg/sftp"
 "golang.org/x/crypto/ssh"
 "github.com/aws/aws-sdk-go/aws"
 "github.com/aws/aws-sdk-go/aws/session"
 "github.com/aws/aws-sdk-go/service/s3/s3manager"
)

func main() {
 // configuração do cliente sftp
 user := "seu_usuario_sftp"
 pass := "sua_senha_sftp"
 host := "endereco_sftp:22"
 config := &ssh.clientconfig{
  user: user,
  auth: []ssh.authmethod{
   ssh.password(pass),
  },
  hostkeycallback: ssh.insecureignorehostkey(),
 }

 // conectar ao servidor sftp
 conn, err := ssh.dial("tcp", host, config)
 if err != nil {
  log.fatal(err)
 }
 client, err := sftp.newclient(conn)
 if err != nil {
  log.fatal(err)
 }
 defer client.close()

 // baixar arquivos do sftp
 remotefilepath := "/path/to/remote/file"
 localdir := "/path/to/local/dir"
 localfilepath := filepath.join(localdir, filepath.base(remotefilepath))
 dstfile, err := os.create(localfilepath)
 if err != nil {
  log.fatal(err)
 }
 defer dstfile.close()

 srcfile, err := client.open(remotefilepath)
 if err != nil {
  log.fatal(err)
 }
 defer srcfile.close()

 if _, err := srcfile.writeto(dstfile); err != nil {
  log.fatal(err)
 }

 fmt.println("arquivo baixado com sucesso:", localfilepath)

 // configuração do cliente s3
 sess := session.must(session.newsession(&aws.config{
  region: aws.string("us-west-2"),
 }))
 uploader := s3manager.newuploader(sess)

 // carregar arquivo para o s3
 file, err := os.open(localfilepath)
 if err != nil {
  log.fatal(err)
 }
 defer file.close()

 _, err = uploader.upload(&s3manager.uploadinput{
  bucket: aws.string("seu-bucket-s3"),
  key:    aws.string(filepath.base(localfilepath)),
  body:   file,
 })
 if err != nil {
  log.fatal("falha ao carregar arquivo para o s3:", err)
 }

 fmt.println("arquivo carregado com sucesso no s3")
}

provider "aws" {
  region = "us-east-1"
}

resource "aws_iam_role" "lambda_execution_role" {
  name = "lambda_execution_role"

  assume_role_policy = jsonencode({
    version = "2012-10-17",
    statement = [
      {
        action = "sts:assumerole",
        effect = "allow",
        principal = {
          service = "lambda.amazonaws.com"
        },
      },
    ]
  })
}

resource "aws_iam_policy" "lambda_policy" {
  name        = "lambda_policy"
  description = "a policy that allows a lambda function to access s3 and sftp resources"

  policy = jsonencode({
    version = "2012-10-17",
    statement = [
      {
        action = [
          "s3:listbucket",
          "s3:getobject",
          "s3:putobject",
        ],
        effect = "allow",
        resource = [
          "arn:aws:s3:::seu-bucket-s3",
          "arn:aws:s3:::seu-bucket-s3/*",
        ],
      },
    ]
  })
}

resource "aws_iam_role_policy_attachment" "lambda_policy_attachment" {
  role       = aws_iam_role.lambda_execution_role.name
  policy_arn = aws_iam_policy.lambda_policy.arn
}

resource "aws_lambda_function" "sftp_lambda" {
  function_name = "sftp_lambda_function"

  s3_bucket = "seu-bucket-s3-com-codigo-lambda"
  s3_key    = "sftp-lambda.zip"

  handler = "main"
  runtime = "go1.x"

  role = aws_iam_role.lambda_execution_role.arn

  environment {
    variables = {
      sftp_host     = "endereco_sftp",
      sftp_user     = "seu_usuario_sftp",
      sftp_password = "sua_senha_sftp",
      s3_bucket     = "seu-bucket-s3",
    }
  }
}

resource "aws_s3_bucket" "s3_bucket" {
  bucket = "seu-bucket-s3"
  acl    = "private"
}

stages:
  - test
  - build
  - deploy

variables:
  s3_bucket: "seu-bucket-s3"
  aws_default_region: "us-east-1"
  tf_version: "1.0.0"

before_script:
  - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )'
  - eval $(ssh-agent -s)
  - echo "$private_key" | tr -d '\r' | ssh-add -
  - mkdir -p ~/.ssh
  - chmod 700 ~/.ssh
  - ssh-keyscan -h 'endereco_sftp' >> ~/.ssh/known_hosts

test:
  stage: test
  image: golang:1.18
  script:
    - go test -v ./...

build:
  stage: build
  image: golang:1.18
  script:
    - go build -o myapp
    - zip -r sftp-lambda.zip myapp
  artifacts:
    paths:
      - sftp-lambda.zip
  only:
    - master

deploy:
  stage: deploy
  image: hashicorp/terraform:$tf_version
  script:
    - terraform init
    - terraform apply -auto-approve
  only:
    - master
  environment:
    name: production

package main

import (
 "bytes"
 "encoding/json"
 "fmt"
 "net/http"
)

// estrutura para a requisição de iniciar um job no databricks
type databricksjobrequest struct {
 jobid int `json:"job_id"`
}

// função para acionar um job no databricks
func triggerdatabricksjob(databricksinstance string, token string, jobid int) error {
 url := fmt.sprintf("https://%s/api/2.0/jobs/run-now", databricksinstance)
 requestbody, _ := json.marshal(databricksjobrequest{jobid: jobid})
 req, err := http.newrequest("post", url, bytes.newbuffer(requestbody))
 if err != nil {
  return err
 }

 req.header.set("content-type", "application/json")
 req.header.set("authorization", fmt.sprintf("bearer %s", token))

 client := &http.client{}
 resp, err := client.do(req)
 if err != nil {
  return err
 }
 defer resp.body.close()

 if resp.statuscode != http.statusok {
  return fmt.errorf("failed to trigger databricks job, status code: %d", resp.statuscode)
 }

 return nil
}

func main() {
 // ... (código existente para conectar ao sftp e carregar no s3)

 // substitua pelos seus valores reais
 databricksinstance := "your-databricks-instance"
 databrickstoken := "your-databricks-token"
 databricksjobid := 123 // id do job que você deseja acionar

 // acionar o job no databricks após o upload para o s3
 err := triggerdatabricksjob(databricksinstance, databrickstoken, databricksjobid)
 if err != nil {
  log.fatal("erro ao acionar o job do databricks:", err)
 }

 fmt.println("job do databricks acionado com sucesso")
}

  git add .
  git commit -m "adiciona função lambda para integração sftp, s3 e databricks"
  git push origin master

git add .
git commit -m "adiciona função lambda para integração sftp, s3 e databricks"
git push origin master
´´´