swarm/api/http: refactored http package (#17309)

swarm/api/api.go

@@ -339,8 +339,7 @@ func (a *API) Get(ctx context.Context, manifestAddr storage.Address, path string
 	if err != nil {
 		apiGetNotFound.Inc(1)
 		status = http.StatusNotFound
-		log.Warn(fmt.Sprintf("loadManifestTrie error: %v", err))
-		return
+		return nil, "", http.StatusNotFound, nil, err
 	}
 
 	log.Debug("trie getting entry", "key", manifestAddr, "path", path)

swarm/api/http/middleware.go

+package http
+
+import (
+	"fmt"
+	"net/http"
+	"runtime/debug"
+	"strings"
+
+	"github.com/ethereum/go-ethereum/metrics"
+	"github.com/ethereum/go-ethereum/swarm/api"
+	"github.com/ethereum/go-ethereum/swarm/log"
+	"github.com/ethereum/go-ethereum/swarm/spancontext"
+	"github.com/pborman/uuid"
+)
+
+// Adapt chains h (main request handler) main handler to adapters (middleware handlers)
+// Please note that the order of execution for `adapters` is FIFO (adapters[0] will be executed first)
+func Adapt(h http.Handler, adapters ...Adapter) http.Handler {
+	for i := range adapters {
+		adapter := adapters[len(adapters)-1-i]
+		h = adapter(h)
+	}
+	return h
+}
+
+type Adapter func(http.Handler) http.Handler
+
+func SetRequestID(h http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		r = r.WithContext(SetRUID(r.Context(), uuid.New()[:8]))
+		metrics.GetOrRegisterCounter(fmt.Sprintf("http.request.%s", r.Method), nil).Inc(1)
+		log.Info("created ruid for request", "ruid", GetRUID(r.Context()), "method", r.Method, "url", r.RequestURI)
+
+		h.ServeHTTP(w, r)
+	})
+}
+
+func ParseURI(h http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		uri, err := api.Parse(strings.TrimLeft(r.URL.Path, "/"))
+		if err != nil {
+			w.WriteHeader(http.StatusBadRequest)
+			RespondError(w, r, fmt.Sprintf("invalid URI %q", r.URL.Path), http.StatusBadRequest)
+			return
+		}
+		if uri.Addr != "" && strings.HasPrefix(uri.Addr, "0x") {
+			uri.Addr = strings.TrimPrefix(uri.Addr, "0x")
+
+			msg := fmt.Sprintf(`The requested hash seems to be prefixed with '0x'. You will be redirected to the correct URL within 5 seconds.<br/>
+			Please click <a href='%[1]s'>here</a> if your browser does not redirect you within 5 seconds.<script>setTimeout("location.href='%[1]s';",5000);</script>`, "/"+uri.String())
+			w.WriteHeader(http.StatusNotFound)
+			w.Write([]byte(msg))
+			return
+		}
+
+		ctx := r.Context()
+		r = r.WithContext(SetURI(ctx, uri))
+		log.Debug("parsed request path", "ruid", GetRUID(r.Context()), "method", r.Method, "uri.Addr", uri.Addr, "uri.Path", uri.Path, "uri.Scheme", uri.Scheme)
+
+		h.ServeHTTP(w, r)
+	})
+}
+
+func InitLoggingResponseWriter(h http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		writer := newLoggingResponseWriter(w)
+
+		h.ServeHTTP(writer, r)
+	})
+}
+
+func InstrumentOpenTracing(h http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		uri := GetURI(r.Context())
+		if uri == nil || r.Method == "" || (uri != nil && uri.Scheme == "") {
+			h.ServeHTTP(w, r) // soft fail
+			return
+		}
+		spanName := fmt.Sprintf("http.%s.%s", r.Method, uri.Scheme)
+		ctx, sp := spancontext.StartSpan(r.Context(), spanName)
+		defer sp.Finish()
+		h.ServeHTTP(w, r.WithContext(ctx))
+	})
+}
+
+func RecoverPanic(h http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		defer func() {
+			if err := recover(); err != nil {
+				log.Error("panic recovery!", "stack trace", debug.Stack(), "url", r.URL.String(), "headers", r.Header)
+			}
+		}()
+		h.ServeHTTP(w, r)
+	})
+}

swarm/api/http/error.go → swarm/api/http/response.go

@@ -14,10 +14,6 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-/*
-Show nicely (but simple) formatted HTML error pages (or respond with JSON
-if the appropriate `Accept` header is set)) for the http package.
-*/
 package http
 
 import (
@@ -31,161 +27,91 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/metrics"
 	"github.com/ethereum/go-ethereum/swarm/api"
-	l "github.com/ethereum/go-ethereum/swarm/log"
 )
 
-//templateMap holds a mapping of an HTTP error code to a template
-var templateMap map[int]*template.Template
-var caseErrors []CaseError
-
 //metrics variables
 var (
-	htmlCounter = metrics.NewRegisteredCounter("api.http.errorpage.html.count", nil)
-	jsonCounter = metrics.NewRegisteredCounter("api.http.errorpage.json.count", nil)
+	htmlCounter      = metrics.NewRegisteredCounter("api.http.errorpage.html.count", nil)
+	jsonCounter      = metrics.NewRegisteredCounter("api.http.errorpage.json.count", nil)
+	plaintextCounter = metrics.NewRegisteredCounter("api.http.errorpage.plaintext.count", nil)
 )
 
 //parameters needed for formatting the correct HTML page
 type ResponseParams struct {
-	Msg       string
+	Msg       template.HTML
 	Code      int
 	Timestamp string
 	template  *template.Template
 	Details   template.HTML
 }
 
-//a custom error case struct that would be used to store validators and
-//additional error info to display with client responses.
-type CaseError struct {
-	Validator func(*Request) bool
-	Msg       func(*Request) string
-}
-
-//we init the error handling right on boot time, so lookup and http response is fast
-func init() {
-	initErrHandling()
-}
-
-func initErrHandling() {
-	//pages are saved as strings - get these strings
-	genErrPage := GetGenericErrorPage()
-	notFoundPage := GetNotFoundErrorPage()
-	multipleChoicesPage := GetMultipleChoicesErrorPage()
-	//map the codes to the available pages
-	tnames := map[int]string{
-		0: genErrPage, //default
-		http.StatusBadRequest:          genErrPage,
-		http.StatusNotFound:            notFoundPage,
-		http.StatusMultipleChoices:     multipleChoicesPage,
-		http.StatusInternalServerError: genErrPage,
-	}
-	templateMap = make(map[int]*template.Template)
-	for code, tname := range tnames {
-		//assign formatted HTML to the code
-		templateMap[code] = template.Must(template.New(fmt.Sprintf("%d", code)).Parse(tname))
-	}
-
-	caseErrors = []CaseError{
-		{
-			Validator: func(r *Request) bool { return r.uri != nil && r.uri.Addr != "" && strings.HasPrefix(r.uri.Addr, "0x") },
-			Msg: func(r *Request) string {
-				uriCopy := r.uri
-				uriCopy.Addr = strings.TrimPrefix(uriCopy.Addr, "0x")
-				return fmt.Sprintf(`The requested hash seems to be prefixed with '0x'. You will be redirected to the correct URL within 5 seconds.<br/>
-			Please click <a href='%[1]s'>here</a> if your browser does not redirect you.<script>setTimeout("location.href='%[1]s';",5000);</script>`, "/"+uriCopy.String())
-			},
-		}}
-}
-
-//ValidateCaseErrors is a method that process the request object through certain validators
-//that assert if certain conditions are met for further information to log as an error
-func ValidateCaseErrors(r *Request) string {
-	for _, err := range caseErrors {
-		if err.Validator(r) {
-			return err.Msg(r)
-		}
-	}
-
-	return ""
-}
-
 //ShowMultipeChoices is used when a user requests a resource in a manifest which results
 //in ambiguous results. It returns a HTML page with clickable links of each of the entry
 //in the manifest which fits the request URI ambiguity.
 //For example, if the user requests bzz:/<hash>/read and that manifest contains entries
 //"readme.md" and "readinglist.txt", a HTML page is returned with this two links.
 //This only applies if the manifest has no default entry
-func ShowMultipleChoices(w http.ResponseWriter, req *Request, list api.ManifestList) {
+func ShowMultipleChoices(w http.ResponseWriter, r *http.Request, list api.ManifestList) {
+	log.Debug("ShowMultipleChoices", "ruid", GetRUID(r.Context()), "uri", GetURI(r.Context()))
 	msg := ""
 	if list.Entries == nil {
-		Respond(w, req, "Could not resolve", http.StatusInternalServerError)
-		return
-	}
-	//make links relative
-	//requestURI comes with the prefix of the ambiguous path, e.g. "read" for "readme.md" and "readinglist.txt"
-	//to get clickable links, need to remove the ambiguous path, i.e. "read"
-	idx := strings.LastIndex(req.RequestURI, "/")
-	if idx == -1 {
-		Respond(w, req, "Internal Server Error", http.StatusInternalServerError)
+		RespondError(w, r, "Could not resolve", http.StatusInternalServerError)
 		return
 	}
-	//remove ambiguous part
-	base := req.RequestURI[:idx+1]
-	for _, e := range list.Entries {
-		//create clickable link for each entry
-		msg += "<a href='" + base + e.Path + "'>" + e.Path + "</a><br/>"
-	}
-	Respond(w, req, msg, http.StatusMultipleChoices)
-}
+	requestUri := strings.TrimPrefix(r.RequestURI, "/")
 
-//Respond is used to show an HTML page to a client.
-//If there is an `Accept` header of `application/json`, JSON will be returned instead
-//The function just takes a string message which will be displayed in the error page.
-//The code is used to evaluate which template will be displayed
-//(and return the correct HTTP status code)
-func Respond(w http.ResponseWriter, req *Request, msg string, code int) {
-	additionalMessage := ValidateCaseErrors(req)
-	switch code {
-	case http.StatusInternalServerError:
-		log.Output(msg, log.LvlError, l.CallDepth, "ruid", req.ruid, "code", code)
-	case http.StatusMultipleChoices:
-		log.Output(msg, log.LvlDebug, l.CallDepth, "ruid", req.ruid, "code", code)
-		listURI := api.URI{
-			Scheme: "bzz-list",
-			Addr:   req.uri.Addr,
-			Path:   req.uri.Path,
-		}
-		additionalMessage = fmt.Sprintf(`<a href="/%s">multiple choices</a>`, listURI.String())
-	default:
-		log.Output(msg, log.LvlDebug, l.CallDepth, "ruid", req.ruid, "code", code)
+	uri, err := api.Parse(requestUri)
+	if err != nil {
+		RespondError(w, r, "Bad Request", http.StatusBadRequest)
 	}
 
-	if code >= 400 {
-		w.Header().Del("Cache-Control") //avoid sending cache headers for errors!
-		w.Header().Del("ETag")
-	}
+	uri.Scheme = "bzz-list"
+	//request the same url just with bzz-list
+	msg += fmt.Sprintf("Disambiguation:<br/>Your request may refer to multiple choices.<br/>Click <a class=\"orange\" href='"+"/"+uri.String()+"'>here</a> if your browser does not redirect you within 5 seconds.<script>setTimeout(\"location.href='%s';\",5000);</script><br/>", "/"+uri.String())
+	RespondTemplate(w, r, "error", msg, http.StatusMultipleChoices)
+}
 
-	respond(w, &req.Request, &ResponseParams{
+func RespondTemplate(w http.ResponseWriter, r *http.Request, templateName, msg string, code int) {
+	log.Debug("RespondTemplate", "ruid", GetRUID(r.Context()), "uri", GetURI(r.Context()))
+	respond(w, r, &ResponseParams{
 		Code:      code,
-		Msg:       msg,
-		Details:   template.HTML(additionalMessage),
+		Msg:       template.HTML(msg),
 		Timestamp: time.Now().Format(time.RFC1123),
-		template:  getTemplate(code),
+		template:  TemplatesMap[templateName],
 	})
 }
 
+func RespondError(w http.ResponseWriter, r *http.Request, msg string, code int) {
+	log.Debug("RespondError", "ruid", GetRUID(r.Context()), "uri", GetURI(r.Context()))
+	RespondTemplate(w, r, "error", msg, code)
+}
+
 //evaluate if client accepts html or json response
 func respond(w http.ResponseWriter, r *http.Request, params *ResponseParams) {
 	w.WriteHeader(params.Code)
-	if r.Header.Get("Accept") == "application/json" {
-		respondJSON(w, params)
+
+	if params.Code >= 400 {
+		w.Header().Del("Cache-Control") //avoid sending cache headers for errors!
+		w.Header().Del("ETag")
+	}
+
+	acceptHeader := r.Header.Get("Accept")
+	// this cannot be in a switch form since an Accept header can be in the form of "Accept: */*, text/html, application/xhtml+xml, application/xml;q=0.9, */*;q=0.8"
+	if strings.Contains(acceptHeader, "application/json") {
+		if err := respondJSON(w, r, params); err != nil {
+			RespondError(w, r, "Internal server error", http.StatusInternalServerError)
+		}
+	} else if strings.Contains(acceptHeader, "text/html") {
+		respondHTML(w, r, params)
 	} else {
-		respondHTML(w, params)
+		respondPlaintext(w, r, params) //returns nice errors for curl
 	}
 }
 
 //return a HTML page
-func respondHTML(w http.ResponseWriter, params *ResponseParams) {
+func respondHTML(w http.ResponseWriter, r *http.Request, params *ResponseParams) {
 	htmlCounter.Inc(1)
+	log.Debug("respondHTML", "ruid", GetRUID(r.Context()))
 	err := params.template.Execute(w, params)
 	if err != nil {
 		log.Error(err.Error())
@@ -193,16 +119,21 @@ func respondHTML(w http.ResponseWriter, params *ResponseParams) {
 }
 
 //return JSON
-func respondJSON(w http.ResponseWriter, params *ResponseParams) {
+func respondJSON(w http.ResponseWriter, r *http.Request, params *ResponseParams) error {
 	jsonCounter.Inc(1)
+	log.Debug("respondJSON", "ruid", GetRUID(r.Context()))
 	w.Header().Set("Content-Type", "application/json")
-	json.NewEncoder(w).Encode(params)
+	return json.NewEncoder(w).Encode(params)
 }
 
-//get the HTML template for a given code
-func getTemplate(code int) *template.Template {
-	if val, tmpl := templateMap[code]; tmpl {
-		return val
-	}
-	return templateMap[0]
+//return plaintext
+func respondPlaintext(w http.ResponseWriter, r *http.Request, params *ResponseParams) error {
+	plaintextCounter.Inc(1)
+	log.Debug("respondPlaintext", "ruid", GetRUID(r.Context()))
+	w.Header().Set("Content-Type", "text/plain")
+	strToWrite := "Code: " + fmt.Sprintf("%d", params.Code) + "\n"
+	strToWrite += "Message: " + string(params.Msg) + "\n"
+	strToWrite += "Timestamp: " + params.Timestamp + "\n"
+	_, err := w.Write([]byte(strToWrite))
+	return err
 }

swarm/api/http/error_test.go → swarm/api/http/response_test.go

@@ -44,7 +44,7 @@ func TestError(t *testing.T) {
 	defer resp.Body.Close()
 	respbody, err = ioutil.ReadAll(resp.Body)
 
-	if resp.StatusCode != 400 && !strings.Contains(string(respbody), "Invalid URI "/this_should_fail_as_no_bzz_protocol_present": unknown scheme") {
+	if resp.StatusCode != 404 && !strings.Contains(string(respbody), "Invalid URI "/this_should_fail_as_no_bzz_protocol_present": unknown scheme") {
 		t.Fatalf("Response body does not match, expected: %v, to contain: %v; received code %d, expected code: %d", string(respbody), "Invalid bzz URI: unknown scheme", 400, resp.StatusCode)
 	}
 

swarm/api/http/sctx.go

+package http
+
+import (
+	"context"
+
+	"github.com/ethereum/go-ethereum/swarm/api"
+	"github.com/ethereum/go-ethereum/swarm/sctx"
+)
+
+type contextKey int
+
+const (
+	uriKey contextKey = iota
+)
+
+func GetRUID(ctx context.Context) string {
+	v, ok := ctx.Value(sctx.HTTPRequestIDKey).(string)
+	if ok {
+		return v
+	}
+	return "xxxxxxxx"
+}
+
+func SetRUID(ctx context.Context, ruid string) context.Context {
+	return context.WithValue(ctx, sctx.HTTPRequestIDKey, ruid)
+}
+
+func GetURI(ctx context.Context) *api.URI {
+	v, ok := ctx.Value(uriKey).(*api.URI)
+	if ok {
+		return v
+	}
+	return nil
+}
+
+func SetURI(ctx context.Context, uri *api.URI) context.Context {
+	return context.WithValue(ctx, uriKey, uri)
+}

swarm/sctx/sctx.go

+package sctx
+
+type ContextKey int
+
+const (
+	HTTPRequestIDKey ContextKey = iota
+)