package toolbox import ( "bytes" "crypto/rand" "encoding/json" "encoding/xml" "errors" "fmt" "io" "log" "net/http" "os" "path" "path/filepath" "regexp" "strings" ) // randomStringSource is the source for generating random strings. const randomStringSource = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0987654321_+" // defaultMaxUpload is the default max upload size (10 mb) const defaultMaxUpload = 10485760 // Tools is the type for this package. Create a variable of this type, and you have access // to all the exported methods with the receiver type *Tools. type Tools struct { MaxJSONSize int // maximum size of JSON file we'll process MaxXMLSize int // maximum size of XML file we'll process MaxFileSize int // maximum size of uploaded files in bytes AllowedFileTypes []string // allowed file types for upload (e.g. image/jpeg) AllowUnknownFields bool // if set to true, allow unknown fields in JSON ErrorLog *log.Logger // the info log. InfoLog *log.Logger // the error log. } // New returns a new toolbox with sensible defaults. func New() Tools { return Tools{ MaxJSONSize: defaultMaxUpload, MaxXMLSize: defaultMaxUpload, MaxFileSize: defaultMaxUpload, InfoLog: log.New(os.Stdout, "INFO\t", log.Ldate|log.Ltime), ErrorLog: log.New(os.Stdout, "ERROR\t", log.Ldate|log.Ltime|log.Lshortfile), } } // JSONResponse is the type used for sending JSON around. type JSONResponse struct { Error bool `json:"error"` Message string `json:"message"` Data interface{} `json:"data,omitempty"` } // XMLResponse is the type used for sending XML around. type XMLResponse struct { Error bool `xml:"error"` Message string `xml:"message"` Data interface{} `xml:"data,omitempty"` } // ReadJSON tries to read the body of a request and converts it from JSON to a variable. The third parameter, data, // is expected to be a pointer, so that we can read data into it. func (t *Tools) ReadJSON(w http.ResponseWriter, r *http.Request, data interface{}) error { // Check content-type header; it should be application/json. If it's not specified, // try to decode the body anyway. if r.Header.Get("Content-Type") != "" { contentType := r.Header.Get("Content-Type") if strings.ToLower(contentType) != "application/json" { return errors.New("the Content-Type header is not application/json") } } // Set a sensible default for the maximum payload size. maxBytes := defaultMaxUpload // If MaxJSONSize is set, use that value instead of default. if t.MaxJSONSize != 0 { maxBytes = t.MaxJSONSize } r.Body = http.MaxBytesReader(w, r.Body, int64(maxBytes)) dec := json.NewDecoder(r.Body) // Should we allow unknown fields? if !t.AllowUnknownFields { dec.DisallowUnknownFields() } // Attempt to decode the data, and figure out what the error is, if any, to send back a human-readable // response. err := dec.Decode(data) if err != nil { var syntaxError *json.SyntaxError var unmarshalTypeError *json.UnmarshalTypeError var invalidUnmarshalError *json.InvalidUnmarshalError switch { case errors.As(err, &syntaxError): return fmt.Errorf("body contains badly-formed JSON (at character %d)", syntaxError.Offset) case errors.Is(err, io.ErrUnexpectedEOF): return errors.New("body contains badly-formed JSON") case errors.As(err, &unmarshalTypeError): return fmt.Errorf("body contains incorrect JSON type for field %q at offset %d", unmarshalTypeError.Field, unmarshalTypeError.Offset) case errors.Is(err, io.EOF): return errors.New("body must not be empty") case strings.HasPrefix(err.Error(), "json: unknown field "): fieldName := strings.TrimPrefix(err.Error(), "json: unknown field ") return fmt.Errorf("body contains unknown key %s", fieldName) case err.Error() == "http: request body too large": return fmt.Errorf("body must not be larger than %d bytes", maxBytes) case errors.As(err, &invalidUnmarshalError): return fmt.Errorf("error unmarshalling json: %s", err.Error()) default: return err } } err = dec.Decode(&struct{}{}) if err != io.EOF { return errors.New("body must only contain a single JSON value") } return nil } // WriteJSON takes a response status code and arbitrary data and writes a JSON response to the client. func (t *Tools) WriteJSON(w http.ResponseWriter, status int, data interface{}, headers ...http.Header) error { out, err := json.Marshal(data) if err != nil { return err } // If we have a value as the last parameter in the function call, then we are setting a custom header. if len(headers) > 0 { for key, value := range headers[0] { w.Header()[key] = value } } // Set the content type and send response. w.Header().Set("Content-Type", "application/json") w.WriteHeader(status) _, _ = w.Write(out) return nil } // ErrorJSON takes an error, and optionally a response status code, and generates and sends // a JSON error response. func (t *Tools) ErrorJSON(w http.ResponseWriter, err error, status ...int) error { statusCode := http.StatusBadRequest // If a custom response code is specified, use that instead of bad request. if len(status) > 0 { statusCode = status[0] } // Build the JSON payload. var payload JSONResponse payload.Error = true payload.Message = err.Error() return t.WriteJSON(w, statusCode, payload) } // RandomString returns a random string of letters of length n, using characters specified in randomStringSource. func (t *Tools) RandomString(n int) string { s, r := make([]rune, n), []rune(randomStringSource) for i := range s { p, _ := rand.Prime(rand.Reader, len(r)) x, y := p.Uint64(), uint64(len(r)) s[i] = r[x%y] } return string(s) } // PushJSONToRemote posts arbitrary json to some url, and returns the response, the response // status code, and error, if any. The final parameter, client, is optional, and will default // to the standard http.Client. It exists to make testing possible without an active remote // url. func (t *Tools) PushJSONToRemote(uri string, data interface{}, client ...*http.Client) (*http.Response, int, error) { // create json we'll send jsonData, err := json.Marshal(data) if err != nil { return nil, 0, err } httpClient := &http.Client{} if len(client) > 0 { httpClient = client[0] } // Build the request and set header. request, err := http.NewRequest("POST", uri, bytes.NewBuffer(jsonData)) if err != nil { return nil, 0, err } request.Header.Set("Content-Type", "application/json") // Call the url. response, err := httpClient.Do(request) if err != nil { return nil, 0, err } defer response.Body.Close() return response, response.StatusCode, nil } // DownloadStaticFile downloads a file to the remote user, and tries to force the browser to avoid displaying it in // the browser window by setting content-disposition. It also allows specification of the display name. func (t *Tools) DownloadStaticFile(w http.ResponseWriter, r *http.Request, p, file, displayName string) { fp := path.Join(p, file) w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", displayName)) http.ServeFile(w, r, fp) } // UploadedFile is the type used for the uploaded file. type UploadedFile struct { NewFileName string OriginalFileName string FileSize int64 } // UploadOneFile is just a convenience method that calls UploadFiles, but expects only one file to // be in the upload. func (t *Tools) UploadOneFile(r *http.Request, uploadDir string, rename ...bool) (*UploadedFile, error) { renameFile := true if len(rename) > 0 { renameFile = rename[0] } files, err := t.UploadFiles(r, uploadDir, renameFile) if err != nil { return nil, err } return files[0], nil } // UploadFiles uploads one or more file to a specified directory, and gives the files a random name. // It returns a slice containing the newly named files, the original file names, the size of the files, // and potentially an error. If the optional last parameter is set to true, then we will not rename // the files, but will use the original file names. func (t *Tools) UploadFiles(r *http.Request, uploadDir string, rename ...bool) ([]*UploadedFile, error) { // check to see if we are renaming the uploadedFiles with the optional last parameter. renameFile := true if len(rename) > 0 { renameFile = rename[0] } var uploadedFiles []*UploadedFile // Create the upload directory if it does not exist. err := t.CreateDirIfNotExist(uploadDir) if err != nil { return nil, err } // Sanity check on t.MaxFileSize. if t.MaxFileSize == 0 { t.MaxFileSize = defaultMaxUpload } // Parse the form, so we have access to the file. Payload is limited to MaxFileSize. err = r.ParseMultipartForm(int64(t.MaxFileSize)) if err != nil { return nil, fmt.Errorf("error parsing form data: %v", err) } for _, fHeaders := range r.MultipartForm.File { for _, hdr := range fHeaders { uploadedFiles, err = func(uploadedFiles []*UploadedFile) ([]*UploadedFile, error) { var uploadedFile UploadedFile infile, err := hdr.Open() if err != nil { return nil, err } defer infile.Close() if hdr.Size > int64(t.MaxFileSize) { return nil, fmt.Errorf("the uploaded file is too big, and must be less than %d", t.MaxFileSize) } buff := make([]byte, 512) _, err = infile.Read(buff) if err != nil { return nil, err } allowed := false filetype := http.DetectContentType(buff) if len(t.AllowedFileTypes) > 0 { for _, x := range t.AllowedFileTypes { if strings.EqualFold(filetype, x) { allowed = true } } } else { allowed = true } if !allowed { return nil, errors.New("the uploaded file type is not permitted") } _, err = infile.Seek(0, 0) if err != nil { fmt.Println(err) return nil, err } if renameFile { uploadedFile.NewFileName = fmt.Sprintf("%s%s", t.RandomString(25), filepath.Ext(hdr.Filename)) } else { uploadedFile.NewFileName = hdr.Filename } uploadedFile.OriginalFileName = hdr.Filename var outfile *os.File defer outfile.Close() if outfile, err = os.Create(filepath.Join(uploadDir, uploadedFile.NewFileName)); nil != err { return nil, err } fileSize, err := io.Copy(outfile, infile) if err != nil { return nil, err } uploadedFile.FileSize = fileSize uploadedFiles = append(uploadedFiles, &uploadedFile) return uploadedFiles, nil }(uploadedFiles) if err != nil { return uploadedFiles, err } } } return uploadedFiles, nil } // CreateDirIfNotExist creates a directory, and all necessary parent directories, if it does not exist. func (t *Tools) CreateDirIfNotExist(path string) error { const mode = 0755 if _, err := os.Stat(path); os.IsNotExist(err) { err := os.MkdirAll(path, mode) if err != nil { return err } } return nil } // Slugify is a (very) simple means of creating a slug from a provided string. func (t *Tools) Slugify(s string) (string, error) { if s == "" { return "", errors.New("empty string not permitted") } var re = regexp.MustCompile(`[^a-z\d]+`) slug := strings.Trim(re.ReplaceAllString(strings.ToLower(s), "-"), "-") if len(slug) == 0 { return "", errors.New("after removing characters, slug is zero length") } return slug, nil } // WriteXML takes a response status code and arbitrary data and writes an XML response to the client. // The Content-Type header is set to application/xml. func (t *Tools) WriteXML(w http.ResponseWriter, status int, data interface{}, headers ...http.Header) error { out, err := xml.Marshal(data) if err != nil { return err } // If we have a value as the last parameter in the function call, then we are setting a custom header. if len(headers) > 0 { for key, value := range headers[0] { w.Header()[key] = value } } // Set the content type and send response. According to RFC 7303, text/xml and application/xml are to be // treated as the same, so we'll just pick one. w.Header().Set("Content-Type", "application/xml") w.WriteHeader(status) // Add the XML header. xmlOut := []byte(xml.Header + string(out)) _, _ = w.Write(xmlOut) return nil } // ReadXML tries to read the body of an XML request into a variable. The third parameter, data, // is expected to be a pointer, so that we can read data into it. func (t *Tools) ReadXML(w http.ResponseWriter, r *http.Request, data interface{}) error { maxBytes := defaultMaxUpload // If MaxXMLSize is set, use that value instead of default. if t.MaxXMLSize != 0 { maxBytes = t.MaxXMLSize } r.Body = http.MaxBytesReader(w, r.Body, int64(maxBytes)) dec := xml.NewDecoder(r.Body) // Attempt to decode the data. err := dec.Decode(data) if err != nil { return err } err = dec.Decode(&struct{}{}) if err != io.EOF { return errors.New("body must only contain a single XML value") } return nil } // ErrorXML takes an error, and optionally a response status code, and generates and sends // an XML error response. func (t *Tools) ErrorXML(w http.ResponseWriter, err error, status ...int) error { statusCode := http.StatusBadRequest // If a custom response code is specified, use that instead of bad request. if len(status) > 0 { statusCode = status[0] } var payload XMLResponse payload.Error = true payload.Message = err.Error() return t.WriteXML(w, statusCode, payload) }