[gitlab] 500: We're sorry, something went wrong on ourend
image.png
检查
[root@test gitaly]# gitlab-ctl status
run: alertmanager: (pid 1130) 1309s; run: log: (pid 1124) 1309s
run: gitaly: (pid 18785) 659s; run: log: (pid 1106) 1309s
run: gitlab-exporter: (pid 1119) 1309s; run: log: (pid 1116) 1309s
run: gitlab-kas: (pid 1133) 1309s; run: log: (pid 1125) 1309s
run: gitlab-workhorse: (pid 1121) 1309s; run: log: (pid 1114) 1309s
run: logrotate: (pid 1135) 1309s; run: log: (pid 1126) 1309s
run: nginx: (pid 1127) 1309s; run: log: (pid 1123) 1309s
run: node-exporter: (pid 1132) 1309s; run: log: (pid 1128) 1309s
run: postgres-exporter: (pid 1117) 1309s; run: log: (pid 1113) 1309s
run: postgresql: (pid 1109) 1309s; run: log: (pid 1105) 1309s
down: prometheus: 1s, normally up, want up; run: log: (pid 1104) 1309s
run: puma: (pid 1134) 1309s; run: log: (pid 1131) 1309s
run: redis: (pid 1120) 1309s; run: log: (pid 1112) 1309s
run: redis-exporter: (pid 1110) 1309s; run: log: (pid 1107) 1309s
run: sidekiq: (pid 1122) 1309s; run: log: (pid 1118) 1309s
[root@test gitaly]#
[root@test gitaly]# cat /var/log/gitlab/gitlab-rails/production.log
puma (6.4.3) lib/puma/request.rb:99:in `handle_request'
puma (6.4.3) lib/puma/server.rb:464:in `process_client'
puma (6.4.3) lib/puma/server.rb:245:in `block in run'
puma (6.4.3) lib/puma/thread_pool.rb:155:in `block in spawn_thread'
ActionView::Template::Error (14:connections to all backends failing; last error: UNKNOWN: unix:/var/opt/gitlab/gitaly/gitaly.socket: No such file or directory. debug_error_string:{UNKNOWN:Error received from peer {grpc_message:"connections to all backends failing; last error: UNKNOWN: unix:/var/opt/gitlab/gitaly/gitaly.socket: No such file or directory", grpc_status:14, created_time:"2024-12-07T21:18:54.087190864+08:00"}}):
23: - if project.creator && use_creator_avatar
24: = render Pajamas::AvatarComponent.new(project.creator, size: 48, alt: '')
25: - else
26: = render Pajamas::AvatarComponent.new(project, size: 48, alt: '')
27:
28: .project-cell{ class: css_class }
29: .project-details.gl-pr-9.gl-sm-pr-0.gl-w-full.gl-flex.gl-flex-col{ data: { testid: 'project-content', qa_project_name: project.name } }
grpc-1.63.0-x86_64 (linux) src/ruby/lib/grpc/generic/active_call.rb:29:in `check_status'
grpc-1.63.0-x86_64 (linux) src/ruby/lib/grpc/generic/active_call.rb:189:in `attach_status_results_and_complete_call'
grpc-1.63.0-x86_64 (linux) src/ruby/lib/grpc/generic/active_call.rb:383:in `request_response'
[root@test gitaly]#
[root@test gitaly]# tail -f /var/log/gitlab/gitaly/current
{"level":"info","msg":"clearing disk cache object folder","pid":30812,"storage":"default","time":"2024-12-07T10:52:24.668Z"}
unclean Gitaly shutdown: disk cache walkers: mkdir /data/gitlab/git-data/repositories/+gitaly/tmp: permission denied
{"level":"warning","msg":"forwarding signal","pid":30806,"process":30812,"signal":17,"time":"2024-12-07T10:52:24.717Z","wrapper":30806}
{"error":"os: process already finished","level":"error","msg":"can't forward the signal","pid":30806,"process":30812,"signal":17,"time":"2024-12-07T10:
52:24.717Z","wrapper":30806}
{"error":"exit status 1","level":"error","msg":"waiting for supervised command","pid":30806,"time":"2024-12-07T10:52:24.717Z","wrapper":30806}
{"level":"error","msg":"wrapper for process shutting down","pid":30806,"process":30812,"time":"2024-12-07T10:52:25.167Z","wrapper":30806}
解决办法
猜测可能是迁移git-data默认存储，没有改好权限问题导致
[root@test gitaly]# chown -R git:git /data/gitlab/git-data/
[root@test gitaly]# ls -la /data/gitlab/git-data/repositories/
total 20
drwxrws--- 4 git git 4096 Dec 5 21:43 .
drwxrws--- 3 git git 4096 Dec 5 21:43 ..
drwx------ 3 git git 4096 Dec 5 21:43 +gitaly
-rw------- 1 git git 64 Dec 5 21:43 .gitaly-metadata
drwx------ 3 git git 4096 Dec 5 21:43 @hashed
[root@test gitaly]#
[root@test gitaly]# gitlab-rake gitlab:gitaly:check
Checking Gitaly ...
Gitaly: ... default ... OK
Checking Gitaly ... Finished
[root@test gitaly]#
[root@test gitaly]# ls -la /var/opt/gitlab/gitaly
total 28
drwx------ 3 git root 4096 Dec 7 21:25 .
drwxr-xr-x 21 root root 4096 Dec 7 20:58 ..
-rw-r----- 1 root git 840 Dec 5 21:53 config.toml
-rw------- 1 git git 5 Dec 7 21:25 gitaly.pid
srwxr-xr-x 1 git git 0 Dec 7 21:25 gitaly.socket
-rw-r--r-- 1 root root 128 Dec 4 22:42 .gitlab_secret
drwx------ 3 git root 4096 Dec 7 21:25 run
-rw-r--r-- 1 root root 72 Dec 4 22:42 VERSION
[root@test gitaly]#
再次检查500消失，搞定！
