From ac384c4e1d207a989d0f646ebc14fd0c26427d4c Mon Sep 17 00:00:00 2001
From: Jason Song <i@wolfogre.com>
Date: Sun, 23 Apr 2023 04:12:41 +0800
Subject: [PATCH] Support upload `outputs` and use `needs` context on Actions
(#24230)
See [Defining outputs for
jobs](https://docs.github.com/en/actions/using-jobs/defining-outputs-for-jobs)
and [Example usage of the needs
context](https://docs.github.com/en/actions/learn-github-actions/contexts#example-usage-of-the-needs-context).
Related to:
- [actions-proto-def
#5](https://gitea.com/gitea/actions-proto-def/pulls/5)
- [act_runner #133](https://gitea.com/gitea/act_runner/pulls/133)
<details>
<summary>Tests & screenshots</summary>
Test workflow file:
```yaml
name: outputs
on: push
jobs:
job1:
runs-on: ubuntu-latest
outputs:
output1: ${{ steps.step1.outputs.output1 }}
output2: ${{ steps.step2.outputs.output2 }}
steps:
- name: step1
id: step1
run: |
date -Is > output1
cat output1
echo "output1=$(cat output1)" >> $GITHUB_OUTPUT
- name: step2
id: step2
run: |
cat /proc/sys/kernel/random/uuid > output2
cat output2
echo "output2=$(cat output2)" >> $GITHUB_OUTPUT
job2:
needs: job1
runs-on: ubuntu-latest
steps:
- run: echo ${{ needs.job1.outputs.output1 }}
- run: echo ${{ needs.job1.outputs.output2 }}
- run: echo ${{ needs.job1.result }}
```
<img width="397" alt="image"
src="https://user-images.githubusercontent.com/9418365/233313322-903e7ebf-49a7-48e2-8c17-95a4581b3284.png">
<img width="385" alt="image"
src="https://user-images.githubusercontent.com/9418365/233313442-30909135-1711-4b78-a5c6-133fcc79f47c.png">
</details>
---------
Co-authored-by: Giteabot <teabot@gitea.io>
---
go.mod | 2 +-
go.sum | 4 +--
models/actions/task_output.go | 51 ++++++++++++++++++++++++++
models/migrations/migrations.go | 2 ++
models/migrations/v1_20/v254.go | 18 ++++++++++
routers/api/actions/runner/runner.go | 28 ++++++++++++++-
routers/api/actions/runner/utils.go | 54 ++++++++++++++++++++++++++++
7 files changed, 155 insertions(+), 4 deletions(-)
create mode 100644 models/actions/task_output.go
create mode 100644 models/migrations/v1_20/v254.go
diff --git a/go.mod b/go.mod
index 92201f8e5e..9f0a23f455 100644
--- a/go.mod
+++ b/go.mod
@@ -3,7 +3,7 @@ module code.gitea.io/gitea
go 1.19
require (
- code.gitea.io/actions-proto-go v0.2.0
+ code.gitea.io/actions-proto-go v0.2.1
code.gitea.io/gitea-vet v0.2.2
code.gitea.io/sdk/gitea v0.15.1
codeberg.org/gusted/mcaptcha v0.0.0-20220723083913-4f3072e1d570
diff --git a/go.sum b/go.sum
index a331ec21e7..e48fa8b45b 100644
--- a/go.sum
+++ b/go.sum
@@ -40,8 +40,8 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl
cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo=
-code.gitea.io/actions-proto-go v0.2.0 h1:nYh9nhhfk67YA4wVNLsCzd//RCvXnljwXClJ33+HPVk=
-code.gitea.io/actions-proto-go v0.2.0/go.mod h1:00ys5QDo1iHN1tHNvvddAcy2W/g+425hQya1cCSvq9A=
+code.gitea.io/actions-proto-go v0.2.1 h1:ToMN/8thz2q10TuCq8dL2d8mI+/pWpJcHCvG+TELwa0=
+code.gitea.io/actions-proto-go v0.2.1/go.mod h1:00ys5QDo1iHN1tHNvvddAcy2W/g+425hQya1cCSvq9A=
code.gitea.io/gitea-vet v0.2.1/go.mod h1:zcNbT/aJEmivCAhfmkHOlT645KNOf9W2KnkLgFjGGfE=
code.gitea.io/gitea-vet v0.2.2 h1:TEOV/Glf38iGmKzKP0EB++Z5OSL4zGg3RrAvlwaMuvk=
code.gitea.io/gitea-vet v0.2.2/go.mod h1:zcNbT/aJEmivCAhfmkHOlT645KNOf9W2KnkLgFjGGfE=
diff --git a/models/actions/task_output.go b/models/actions/task_output.go
new file mode 100644
index 0000000000..5291a783d6
--- /dev/null
+++ b/models/actions/task_output.go
@@ -0,0 +1,51 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package actions
+
+import (
+ "context"
+
+ "code.gitea.io/gitea/models/db"
+)
+
+// ActionTaskOutput represents an output of ActionTask.
+// So the outputs are bound to a task, that means when a completed job has been rerun,
+// the outputs of the job will be reset because the task is new.
+// It's by design, to avoid the outputs of the old task to be mixed with the new task.
+type ActionTaskOutput struct {
+ ID int64
+ TaskID int64 `xorm:"INDEX UNIQUE(task_id_output_key)"`
+ OutputKey string `xorm:"VARCHAR(255) UNIQUE(task_id_output_key)"`
+ OutputValue string `xorm:"MEDIUMTEXT"`
+}
+
+// FindTaskOutputByTaskID returns the outputs of the task.
+func FindTaskOutputByTaskID(ctx context.Context, taskID int64) ([]*ActionTaskOutput, error) {
+ var outputs []*ActionTaskOutput
+ return outputs, db.GetEngine(ctx).Where("task_id=?", taskID).Find(&outputs)
+}
+
+// FindTaskOutputKeyByTaskID returns the keys of the outputs of the task.
+func FindTaskOutputKeyByTaskID(ctx context.Context, taskID int64) ([]string, error) {
+ var keys []string
+ return keys, db.GetEngine(ctx).Table(ActionTaskOutput{}).Where("task_id=?", taskID).Cols("output_key").Find(&keys)
+}
+
+// InsertTaskOutputIfNotExist inserts a new task output if it does not exist.
+func InsertTaskOutputIfNotExist(ctx context.Context, taskID int64, key, value string) error {
+ return db.WithTx(ctx, func(ctx context.Context) error {
+ sess := db.GetEngine(ctx)
+ if exist, err := sess.Exist(&ActionTaskOutput{TaskID: taskID, OutputKey: key}); err != nil {
+ return err
+ } else if exist {
+ return nil
+ }
+ _, err := sess.Insert(&ActionTaskOutput{
+ TaskID: taskID,
+ OutputKey: key,
+ OutputValue: value,
+ })
+ return err
+ })
+}
diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go
index 42806a808f..9de5931d71 100644
--- a/models/migrations/migrations.go
+++ b/models/migrations/migrations.go
@@ -485,6 +485,8 @@ var migrations = []Migration{
NewMigration("Fix incorrect admin team unit access mode", v1_20.FixIncorrectAdminTeamUnitAccessMode),
// v253 -> v254
NewMigration("Fix ExternalTracker and ExternalWiki accessMode in owner and admin team", v1_20.FixExternalTrackerAndExternalWikiAccessModeInOwnerAndAdminTeam),
+ // v254 -> v255
+ NewMigration("Add ActionTaskOutput table", v1_20.AddActionTaskOutputTable),
}
// GetCurrentDBVersion returns the current db version
diff --git a/models/migrations/v1_20/v254.go b/models/migrations/v1_20/v254.go
new file mode 100644
index 0000000000..1e26979a5b
--- /dev/null
+++ b/models/migrations/v1_20/v254.go
@@ -0,0 +1,18 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package v1_20 //nolint
+
+import (
+ "xorm.io/xorm"
+)
+
+func AddActionTaskOutputTable(x *xorm.Engine) error {
+ type ActionTaskOutput struct {
+ ID int64
+ TaskID int64 `xorm:"INDEX UNIQUE(task_id_output_key)"`
+ OutputKey string `xorm:"VARCHAR(255) UNIQUE(task_id_output_key)"`
+ OutputValue string `xorm:"MEDIUMTEXT"`
+ }
+ return x.Sync(new(ActionTaskOutput))
+}
diff --git a/routers/api/actions/runner/runner.go b/routers/api/actions/runner/runner.go
index a445864858..73c6b746a0 100644
--- a/routers/api/actions/runner/runner.go
+++ b/routers/api/actions/runner/runner.go
@@ -97,7 +97,7 @@ func (s *Service) Register(
// FetchTask assigns a task to the runner
func (s *Service) FetchTask(
ctx context.Context,
- req *connect.Request[runnerv1.FetchTaskRequest],
+ _ *connect.Request[runnerv1.FetchTaskRequest],
) (*connect.Response[runnerv1.FetchTaskResponse], error) {
runner := GetRunner(ctx)
@@ -145,6 +145,31 @@ func (s *Service) UpdateTask(
return nil, status.Errorf(codes.Internal, "update task: %v", err)
}
+ for k, v := range req.Msg.Outputs {
+ if len(k) > 255 {
+ log.Warn("Ignore the output of task %d because the key is too long: %q", task.ID, k)
+ continue
+ }
+ // The value can be a maximum of 1 MB
+ if l := len(v); l > 1024*1024 {
+ log.Warn("Ignore the output %q of task %d because the value is too long: %v", k, task.ID, l)
+ continue
+ }
+ // There's another limitation on GitHub that the total of all outputs in a workflow run can be a maximum of 50 MB.
+ // We don't check the total size here because it's not easy to do, and it doesn't really worth it.
+ // See https://docs.github.com/en/actions/using-jobs/defining-outputs-for-jobs
+
+ if err := actions_model.InsertTaskOutputIfNotExist(ctx, task.ID, k, v); err != nil {
+ log.Warn("Failed to insert the output %q of task %d: %v", k, task.ID, err)
+ // It's ok not to return errors, the runner will resend the outputs.
+ }
+ }
+ sentOutputs, err := actions_model.FindTaskOutputKeyByTaskID(ctx, task.ID)
+ if err != nil {
+ log.Warn("Failed to find the sent outputs of task %d: %v", task.ID, err)
+ // It's not to return errors, it can be handled when the runner resends sent outputs.
+ }
+
if err := task.LoadJob(ctx); err != nil {
return nil, status.Errorf(codes.Internal, "load job: %v", err)
}
@@ -162,6 +187,7 @@ func (s *Service) UpdateTask(
Id: req.Msg.State.Id,
Result: task.Status.AsResult(),
},
+ SentOutputs: sentOutputs,
}), nil
}
diff --git a/routers/api/actions/runner/utils.go b/routers/api/actions/runner/utils.go
index cbfcd79a9d..705867a9b1 100644
--- a/routers/api/actions/runner/utils.go
+++ b/routers/api/actions/runner/utils.go
@@ -37,6 +37,17 @@ func pickTask(ctx context.Context, runner *actions_model.ActionRunner) (*runnerv
Context: generateTaskContext(t),
Secrets: getSecretsOfTask(ctx, t),
}
+
+ if needs, err := findTaskNeeds(ctx, t); err != nil {
+ log.Error("Cannot find needs for task %v: %v", t.ID, err)
+ // Go on with empty needs.
+ // If return error, the task will be wild, which means the runner will never get it when it has been assigned to the runner.
+ // In contrast, missing needs is less serious.
+ // And the task will fail and the runner will report the error in the logs.
+ } else {
+ task.Needs = needs
+ }
+
return task, true, nil
}
@@ -124,3 +135,46 @@ func generateTaskContext(t *actions_model.ActionTask) *structpb.Struct {
return taskContext
}
+
+func findTaskNeeds(ctx context.Context, task *actions_model.ActionTask) (map[string]*runnerv1.TaskNeed, error) {
+ if err := task.LoadAttributes(ctx); err != nil {
+ return nil, fmt.Errorf("LoadAttributes: %w", err)
+ }
+ if len(task.Job.Needs) == 0 {
+ return nil, nil
+ }
+ needs := map[string]struct{}{}
+ for _, v := range task.Job.Needs {
+ needs[v] = struct{}{}
+ }
+
+ jobs, _, err := actions_model.FindRunJobs(ctx, actions_model.FindRunJobOptions{RunID: task.Job.RunID})
+ if err != nil {
+ return nil, fmt.Errorf("FindRunJobs: %w", err)
+ }
+
+ ret := make(map[string]*runnerv1.TaskNeed, len(needs))
+ for _, job := range jobs {
+ if _, ok := needs[job.JobID]; !ok {
+ continue
+ }
+ if job.TaskID == 0 || !job.Status.IsDone() {
+ // it shouldn't happen, or the job has been rerun
+ continue
+ }
+ outputs := make(map[string]string)
+ got, err := actions_model.FindTaskOutputByTaskID(ctx, job.TaskID)
+ if err != nil {
+ return nil, fmt.Errorf("FindTaskOutputByTaskID: %w", err)
+ }
+ for _, v := range got {
+ outputs[v.OutputKey] = v.OutputValue
+ }
+ ret[job.JobID] = &runnerv1.TaskNeed{
+ Outputs: outputs,
+ Result: runnerv1.Result(job.Status),
+ }
+ }
+
+ return ret, nil
+}