Fix rate limit and runner registration logic (#309)
* errors.Is compares all members of a struct to return true which never happened * switched to type check instead of exact value check * notRegistered was using double negation in if statement which lead to unregistering runners after the registration timeout
This commit is contained in:
		
							parent
							
								
									34c6c3d9cd
								
							
						
					
					
						commit
						bc8bc70f69
					
				|  | @ -244,16 +244,18 @@ func (r *RunnerReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { | ||||||
| 			return ctrl.Result{}, err | 			return ctrl.Result{}, err | ||||||
| 		} | 		} | ||||||
| 
 | 
 | ||||||
| 		var notRegistered bool | 		notRegistered := false | ||||||
| 
 | 
 | ||||||
| 		runnerBusy, err := r.GitHubClient.IsRunnerBusy(ctx, runner.Spec.Enterprise, runner.Spec.Organization, runner.Spec.Repository, runner.Name) | 		runnerBusy, err := r.GitHubClient.IsRunnerBusy(ctx, runner.Spec.Enterprise, runner.Spec.Organization, runner.Spec.Repository, runner.Name) | ||||||
| 		if err != nil { | 		if err != nil { | ||||||
| 			if errors.Is(err, github.RunnerNotFound{}) { | 			var e *github.RunnerNotFound | ||||||
|  | 			if errors.As(err, &e) { | ||||||
| 				log.Error(err, "Failed to check if runner is busy. Probably this runner has never been successfully registered to GitHub.") | 				log.Error(err, "Failed to check if runner is busy. Probably this runner has never been successfully registered to GitHub.") | ||||||
| 
 | 
 | ||||||
| 				notRegistered = true | 				notRegistered = true | ||||||
| 			} else { | 			} else { | ||||||
| 				if errors.Is(err, &gogithub.RateLimitError{}) { | 				var e *gogithub.RateLimitError | ||||||
|  | 				if errors.As(err, &e) { | ||||||
| 					// We log the underlying error when we failed calling GitHub API to list or unregisters,
 | 					// We log the underlying error when we failed calling GitHub API to list or unregisters,
 | ||||||
| 					// or the runner is still busy.
 | 					// or the runner is still busy.
 | ||||||
| 					log.Error( | 					log.Error( | ||||||
|  | @ -284,7 +286,7 @@ func (r *RunnerReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { | ||||||
| 		currentTime := time.Now() | 		currentTime := time.Now() | ||||||
| 		registrationDidTimeout := currentTime.Sub(pod.CreationTimestamp.Add(registrationTimeout)) > 0 | 		registrationDidTimeout := currentTime.Sub(pod.CreationTimestamp.Add(registrationTimeout)) > 0 | ||||||
| 
 | 
 | ||||||
| 		if !notRegistered && registrationDidTimeout { | 		if notRegistered && registrationDidTimeout { | ||||||
| 			log.Info( | 			log.Info( | ||||||
| 				"Runner failed to register itself to GitHub in timely manner. "+ | 				"Runner failed to register itself to GitHub in timely manner. "+ | ||||||
| 					"Recreating the pod to see if it resolves the issue. "+ | 					"Recreating the pod to see if it resolves the issue. "+ | ||||||
|  |  | ||||||
|  | @ -107,17 +107,15 @@ func (r *RunnerReplicaSetReconciler) Reconcile(req ctrl.Request) (ctrl.Result, e | ||||||
| 		for _, runner := range myRunners { | 		for _, runner := range myRunners { | ||||||
| 			busy, err := r.GitHubClient.IsRunnerBusy(ctx, runner.Spec.Enterprise, runner.Spec.Organization, runner.Spec.Repository, runner.Name) | 			busy, err := r.GitHubClient.IsRunnerBusy(ctx, runner.Spec.Enterprise, runner.Spec.Organization, runner.Spec.Repository, runner.Name) | ||||||
| 			if err != nil { | 			if err != nil { | ||||||
|  | 				notRegistered := false | ||||||
|  | 
 | ||||||
|  | 				var e *github.RunnerNotFound | ||||||
|  | 				if errors.As(err, &e) { | ||||||
| 					log.Error(err, "Failed to check if runner is busy. Probably this runner has never been successfully registered to GitHub, and therefore we prioritize it for deletion", "runnerName", runner.Name) | 					log.Error(err, "Failed to check if runner is busy. Probably this runner has never been successfully registered to GitHub, and therefore we prioritize it for deletion", "runnerName", runner.Name) | ||||||
| 
 |  | ||||||
| 				var notRegistered bool |  | ||||||
| 
 |  | ||||||
| 				if err != nil { |  | ||||||
| 					if errors.Is(err, github.RunnerNotFound{}) { |  | ||||||
| 						log.Error(err, "Failed to check if runner is busy. Probably this runner has never been successfully registered to GitHub.") |  | ||||||
| 
 |  | ||||||
| 					notRegistered = true | 					notRegistered = true | ||||||
| 				} else { | 				} else { | ||||||
| 						if errors.Is(err, &gogithub.RateLimitError{}) { | 					var e *gogithub.RateLimitError | ||||||
|  | 					if errors.As(err, &e) { | ||||||
| 						// We log the underlying error when we failed calling GitHub API to list or unregisters,
 | 						// We log the underlying error when we failed calling GitHub API to list or unregisters,
 | ||||||
| 						// or the runner is still busy.
 | 						// or the runner is still busy.
 | ||||||
| 						log.Error( | 						log.Error( | ||||||
|  | @ -133,13 +131,12 @@ func (r *RunnerReplicaSetReconciler) Reconcile(req ctrl.Request) (ctrl.Result, e | ||||||
| 
 | 
 | ||||||
| 					return ctrl.Result{}, err | 					return ctrl.Result{}, err | ||||||
| 				} | 				} | ||||||
| 				} |  | ||||||
| 
 | 
 | ||||||
| 				registrationTimeout := 15 * time.Minute | 				registrationTimeout := 15 * time.Minute | ||||||
| 				currentTime := time.Now() | 				currentTime := time.Now() | ||||||
| 				registrationDidTimeout := currentTime.Sub(runner.CreationTimestamp.Add(registrationTimeout)) > 0 | 				registrationDidTimeout := currentTime.Sub(runner.CreationTimestamp.Add(registrationTimeout)) > 0 | ||||||
| 
 | 
 | ||||||
| 				if !notRegistered && registrationDidTimeout { | 				if notRegistered && registrationDidTimeout { | ||||||
| 					log.Info( | 					log.Info( | ||||||
| 						"Runner failed to register itself to GitHub in timely manner. "+ | 						"Runner failed to register itself to GitHub in timely manner. "+ | ||||||
| 							"Recreating the pod to see if it resolves the issue. "+ | 							"Recreating the pod to see if it resolves the issue. "+ | ||||||
|  |  | ||||||
|  | @ -287,7 +287,7 @@ type RunnerNotFound struct { | ||||||
| 	runnerName string | 	runnerName string | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| func (e RunnerNotFound) Error() string { | func (e *RunnerNotFound) Error() string { | ||||||
| 	return fmt.Sprintf("runner %q not found", e.runnerName) | 	return fmt.Sprintf("runner %q not found", e.runnerName) | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
|  | @ -303,5 +303,5 @@ func (r *Client) IsRunnerBusy(ctx context.Context, enterprise, org, repo, name s | ||||||
| 		} | 		} | ||||||
| 	} | 	} | ||||||
| 
 | 
 | ||||||
| 	return false, RunnerNotFound{runnerName: name} | 	return false, &RunnerNotFound{runnerName: name} | ||||||
| } | } | ||||||
|  |  | ||||||
		Loading…
	
		Reference in New Issue