From 2f932269a9a7ef7af747ed9ce44d9985bd4a9fb6 Mon Sep 17 00:00:00 2001 From: bbrodriges Date: Thu, 2 Apr 2020 11:42:31 +0000 Subject: [PATCH] lecture 6: http and context --- .../context/cancelation/cancelation.go | 43 ++++ .../06-http/context/httpserver/handler.go | 35 +++ .../context/httpserver/handler_test.go | 18 ++ .../06-http/context/httpserver/httpserver.go | 36 +++ lectures/06-http/customclient/customclient.go | 43 ++++ lectures/06-http/custompost/custompost.go | 31 +++ .../gracefulshutdown/gracefulshutdown.go | 49 ++++ lectures/06-http/httptest/code.go | 47 ++++ lectures/06-http/httptest/code_test.go | 30 +++ .../06-http/keepalive/advanced/advanced.go | 38 +++ lectures/06-http/keepalive/correct/correct.go | 39 ++++ lectures/06-http/keepalive/naive/naive.go | 30 +++ lectures/06-http/lecture.slide | 221 ++++++++++++++++++ lectures/06-http/simpleget/simpleget.go | 14 ++ lectures/06-http/simpleserver/middleware.go | 50 ++++ lectures/06-http/simpleserver/router.go | 33 +++ lectures/06-http/simpleserver/simpleserver.go | 27 +++ 17 files changed, 784 insertions(+) create mode 100644 lectures/06-http/context/cancelation/cancelation.go create mode 100644 lectures/06-http/context/httpserver/handler.go create mode 100644 lectures/06-http/context/httpserver/handler_test.go create mode 100644 lectures/06-http/context/httpserver/httpserver.go create mode 100644 lectures/06-http/customclient/customclient.go create mode 100644 lectures/06-http/custompost/custompost.go create mode 100644 lectures/06-http/gracefulshutdown/gracefulshutdown.go create mode 100644 lectures/06-http/httptest/code.go create mode 100644 lectures/06-http/httptest/code_test.go create mode 100644 lectures/06-http/keepalive/advanced/advanced.go create mode 100644 lectures/06-http/keepalive/correct/correct.go create mode 100644 lectures/06-http/keepalive/naive/naive.go create mode 100644 lectures/06-http/lecture.slide create mode 100644 lectures/06-http/simpleget/simpleget.go create mode 100644 lectures/06-http/simpleserver/middleware.go create mode 100644 lectures/06-http/simpleserver/router.go create mode 100644 lectures/06-http/simpleserver/simpleserver.go diff --git a/lectures/06-http/context/cancelation/cancelation.go b/lectures/06-http/context/cancelation/cancelation.go new file mode 100644 index 0000000..a5f34c2 --- /dev/null +++ b/lectures/06-http/context/cancelation/cancelation.go @@ -0,0 +1,43 @@ +package cancelation + +import ( + "context" + "time" +) + +func SimpleCancelation() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + go func() { + time.Sleep(5 * time.Second) + cancel() + }() + + if err := doSlowJob(ctx); err != nil { + panic(err) + } +} + +func SimpleTimeout() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + if err := doSlowJob(ctx); err != nil { + panic(err) + } +} + +func doSlowJob(ctx context.Context) error { + for { + select { + case <-ctx.Done(): + return ctx.Err() + default: + // perform a portion of slow job + time.Sleep(1 * time.Second) + } + } +} + +// OMIT diff --git a/lectures/06-http/context/httpserver/handler.go b/lectures/06-http/context/httpserver/handler.go new file mode 100644 index 0000000..9a084c4 --- /dev/null +++ b/lectures/06-http/context/httpserver/handler.go @@ -0,0 +1,35 @@ +package httpserver + +import ( + "bufio" + "fmt" + "net/http" + "os" + "time" +) + +type handler struct{} + +func (h handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + reqTime := ctx.Value(ReqTimeContextKey{}).(time.Time) + defer func() { + fmt.Printf("handler finished in %s", time.Since(reqTime)) + }() + + fd, _ := os.Open("core.c") + defer fd.Close() + + scanner := bufio.NewScanner(fd) + + for scanner.Scan() { + select { + case <-ctx.Done(): + fmt.Println(ctx.Err()) + return + default: + _, _ = w.Write(scanner.Bytes()) + } + } +} diff --git a/lectures/06-http/context/httpserver/handler_test.go b/lectures/06-http/context/httpserver/handler_test.go new file mode 100644 index 0000000..3a96425 --- /dev/null +++ b/lectures/06-http/context/httpserver/handler_test.go @@ -0,0 +1,18 @@ +package httpserver + +import ( + "net/http/httptest" + "testing" +) + +func TestHandlerServeHTTP(t *testing.T) { + w := httptest.NewRecorder() + r := httptest.NewRequest("GET", "/", nil) + + h := handler{} + h.ServeHTTP(w, r) + + if w.Code != 200 { + t.Errorf("expected HTTP 200, got: %d", w.Code) + } +} diff --git a/lectures/06-http/context/httpserver/httpserver.go b/lectures/06-http/context/httpserver/httpserver.go new file mode 100644 index 0000000..52ccc03 --- /dev/null +++ b/lectures/06-http/context/httpserver/httpserver.go @@ -0,0 +1,36 @@ +package httpserver + +import ( + "context" + "net" + "net/http" + "os" + "os/signal" + "syscall" + "time" +) + +type ReqTimeContextKey struct{} + +func runServer() error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + go func() { + stop := make(chan os.Signal, 1) + signal.Notify(stop, syscall.SIGINT, syscall.SIGTERM) + <-stop + cancel() + }() + + srv := &http.Server{ + Addr: ":8080", + Handler: handler{}, + BaseContext: func(_ net.Listener) context.Context { + ctx = context.WithValue(ctx, ReqTimeContextKey{}, time.Now()) + return ctx + }, + } + + return srv.ListenAndServe() +} diff --git a/lectures/06-http/customclient/customclient.go b/lectures/06-http/customclient/customclient.go new file mode 100644 index 0000000..79a0b74 --- /dev/null +++ b/lectures/06-http/customclient/customclient.go @@ -0,0 +1,43 @@ +package main + +import ( + "crypto/tls" + "crypto/x509" + "errors" + "net/http" + "net/http/cookiejar" + "time" +) + +func main() { + // все куки записанные в этот Jar будут передаваться + // и изменяться во всех запросах + cj, _ := cookiejar.New(nil) + + client := &http.Client{ + Timeout: 1 * time.Second, + Jar: cj, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + if len(via) > 20 { + return errors.New("too many redirects") + } + return nil + }, + Transport: &http.Transport{ + // резмер буферов чтения и записи (4KB по-умолчанию) + WriteBufferSize: 32 << 10, + ReadBufferSize: 32 << 10, + // конфиг работы с зашифрованными соединениями + TLSClientConfig: &tls.Config{ + Certificates: []tls.Certificate{}, + RootCAs: &x509.CertPool{}, + // только для отладки! + InsecureSkipVerify: true, + // .. + }, + // ... + }, + } + + _ = client +} diff --git a/lectures/06-http/custompost/custompost.go b/lectures/06-http/custompost/custompost.go new file mode 100644 index 0000000..824988d --- /dev/null +++ b/lectures/06-http/custompost/custompost.go @@ -0,0 +1,31 @@ +package main + +import ( + "bytes" + "fmt" + "log" + "net/http" + "net/http/httputil" +) + +func main() { + body := bytes.NewBufferString("All your base are belong to us") + req, err := http.NewRequest("POST", "https://myapi.com/create", body) + if err != nil { + log.Fatal(err) + } + + req.Header.Set("X-Source", "Zero Wing") + + repr, err := httputil.DumpRequestOut(req, true) + if err == nil { + fmt.Println(string(repr)) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + log.Fatal(err) + } + + fmt.Println(resp.StatusCode) +} diff --git a/lectures/06-http/gracefulshutdown/gracefulshutdown.go b/lectures/06-http/gracefulshutdown/gracefulshutdown.go new file mode 100644 index 0000000..05d64b5 --- /dev/null +++ b/lectures/06-http/gracefulshutdown/gracefulshutdown.go @@ -0,0 +1,49 @@ +package main + +import ( + "context" + "fmt" + "log" + "net/http" + "os" + "os/signal" + "syscall" + "time" +) + +func main() { + if err := run(); err != nil { + log.Fatal(err) + } +} + +func run() error { + handler := func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("pong")) + } + + srv := &http.Server{ + Addr: ":8080", + Handler: http.HandlerFunc(handler), + } + + serveChan := make(chan error, 1) + go func() { + serveChan <- srv.ListenAndServe() + }() + + stop := make(chan os.Signal, 1) + signal.Notify(stop, syscall.SIGINT, syscall.SIGTERM) + + select { + case <-stop: + fmt.Println("shutting down gracefully") + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + return srv.Shutdown(ctx) + case err := <-serveChan: + return err + } +} diff --git a/lectures/06-http/httptest/code.go b/lectures/06-http/httptest/code.go new file mode 100644 index 0000000..cab3dca --- /dev/null +++ b/lectures/06-http/httptest/code.go @@ -0,0 +1,47 @@ +package httptest + +import ( + "context" + "io/ioutil" + "net/http" + "strconv" +) + +const ( + BaseURLProd = "https://github.com/api" +) + +type APIClient struct { + baseURL string + httpc *http.Client +} + +func NewAPICLient(baseURL string) *APIClient { + if baseURL == "" { + baseURL = BaseURLProd + } + return &APIClient{ + baseURL: baseURL, + httpc: new(http.Client), + } +} + +func (c *APIClient) GetReposCount(ctx context.Context, userID string) (int, error) { + url := c.baseURL + "/users/" + userID + "/repos/count" + req, _ := http.NewRequestWithContext(ctx, "GET", url, nil) + + resp, err := c.httpc.Do(req) + if err != nil { + return 0, err + } + defer resp.Body.Close() + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return 0, err + } + + return strconv.Atoi(string(body)) +} + +// OMIT diff --git a/lectures/06-http/httptest/code_test.go b/lectures/06-http/httptest/code_test.go new file mode 100644 index 0000000..8c1edf9 --- /dev/null +++ b/lectures/06-http/httptest/code_test.go @@ -0,0 +1,30 @@ +package httptest + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" +) + +func TestGetReposCount(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("42")) + })) + + defer srv.Close() + + client := NewAPICLient(srv.URL) + count, err := client.GetReposCount(context.Background(), "007") + + if err != nil { + t.Errorf("unexpected error: %s", err) + } + + expectedCount := 42 + if count != expectedCount { + t.Errorf("expected count to be: %d, got: %d", expectedCount, count) + } +} + +// OMIT diff --git a/lectures/06-http/keepalive/advanced/advanced.go b/lectures/06-http/keepalive/advanced/advanced.go new file mode 100644 index 0000000..1c2ef50 --- /dev/null +++ b/lectures/06-http/keepalive/advanced/advanced.go @@ -0,0 +1,38 @@ +package main + +import ( + "crypto/tls" + "fmt" + "net/http" + "sync" +) + +func main() { + urls := []string{"https://golang.org/doc", "https://golang.org/pkg", "https://golang.org/help"} + + client := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + }, + } + + var wg sync.WaitGroup + + for _, url := range urls { + wg.Add(1) + + go func(url string) { + defer wg.Done() + resp, err := client.Get(url) + if err != nil { + fmt.Printf("%s: %s\n", url, err) + return + } + fmt.Printf("%s - %d\n", url, resp.StatusCode) + }(url) + } + + wg.Wait() +} diff --git a/lectures/06-http/keepalive/correct/correct.go b/lectures/06-http/keepalive/correct/correct.go new file mode 100644 index 0000000..3503a57 --- /dev/null +++ b/lectures/06-http/keepalive/correct/correct.go @@ -0,0 +1,39 @@ +package main + +import ( + "fmt" + "io" + "io/ioutil" + "net/http" + "sync" +) + +func main() { + urls := []string{"https://golang.org/doc", "https://golang.org/pkg", "https://golang.org/help"} + + client := &http.Client{ + Transport: &http.Transport{ + MaxConnsPerHost: 100, + }, + } + + var wg sync.WaitGroup + + for _, url := range urls { + wg.Add(1) + + go func(url string) { + defer wg.Done() + resp, err := client.Get(url) + if err != nil { + fmt.Printf("%s: %s\n", url, err) + return + } + defer resp.Body.Close() + _, _ = io.Copy(ioutil.Discard, resp.Body) + fmt.Printf("%s - %d\n", url, resp.StatusCode) + }(url) + } + + wg.Wait() +} diff --git a/lectures/06-http/keepalive/naive/naive.go b/lectures/06-http/keepalive/naive/naive.go new file mode 100644 index 0000000..e244cbe --- /dev/null +++ b/lectures/06-http/keepalive/naive/naive.go @@ -0,0 +1,30 @@ +package main + +import ( + "fmt" + "net/http" + "sync" +) + +func main() { + urls := []string{"https://golang.org/doc", "https://golang.org/pkg", "https://golang.org/help"} + + var wg sync.WaitGroup + + for _, url := range urls { + wg.Add(1) + + go func(url string) { + defer wg.Done() + var client http.Client + resp, err := client.Get(url) + if err != nil { + fmt.Printf("%s: %s\n", url, err) + return + } + fmt.Printf("%s - %d\n", url, resp.StatusCode) + }(url) + } + + wg.Wait() +} diff --git a/lectures/06-http/lecture.slide b/lectures/06-http/lecture.slide new file mode 100644 index 0000000..4b10f45 --- /dev/null +++ b/lectures/06-http/lecture.slide @@ -0,0 +1,221 @@ +http и context +Лекция 6 + +Георгий Зуйков + +* Имеем из коробки + +- HTTP клиент (HTTP/1.x, HTTP/2) +- HTTP сервер (с поддержкой TLS) +- Тестирование + +* net/http + +* net/http + +Содержит в себе: +- HTTP клиент и сервер +- Константы статусов и мотодов HTTP +- Sentinel ошибки +- Вспомогательные функции для составления и разбора HTTP запросов + +* HTTP клиент + +* Делаем запрос + +.play simpleget/simpleget.go + +Доступные функции: + + Get(url string) (*Response, error) + Post(url, contentType string, body io.Reader) (*Response, error) + Head(url string) (*Response, error) + PostForm(url string, form url.Values) (*Response, error) + +* Делаем более лучший запрос + +.play custompost/custompost.go + +`http.DefaultClient` - базовый глобальный клиент с настройками по-умолчанию. + +* http.Client + + type Client struct { + // Определяет механизм выполнения каждого запроса + Transport RoundTripper + + // Функция для кастомной проверки редиректов + // По-умолчанию - максимум 10 редиректов + CheckRedirect func(req *Request, via []*Request) error + + // Хранилище кук + Jar CookieJar + + // Таймаут любого запроса от клиента + // Считается все время от соединения до конца вычитывания тела + // 0 - без таймаута + Timeout time.Duration + } + +* Тонкая настройка клиента + +.code customclient/customclient.go /func main/,/^}/ + +* Keepalive + +.play keepalive/naive/naive.go /func main/,/^}/ + +Как-то медленно + +* Keepalive + +.play keepalive/advanced/advanced.go /func main/,/^}/ + +Что-то лыжи не едут + +* Keepalive + +.play keepalive/correct/correct.go /func main/,/^}/ + +Вот теперь всё как надо! + +* HTTP сервер + +* Простой HTTP сервер + +.code simpleserver/simpleserver.go /func RunServer/,/^}/ + +.code simpleserver/simpleserver.go /func RunTLSServer/,/^}/ + +http.Handler - интерфейс, описывающий функцию для обработки HTTP запроса. + + type Handler interface { + ServeHTTP(ResponseWriter, *Request) + } + +* Роутинг + +.code simpleserver/router.go /func RunServerWithRouting/,/OMIT/ + +* Что нужно знать + +- Запуск сервера - блокирующая операция +- Каждый входящий HTTP запрос обрабатывается в отдельной горутине (следите за дескрипторами) +- Паника внутри одного хэндлера не приводит к остановке всего сервера +- Неотловленная паника закрывает HTTP соединение +- Хедеры ответа нельзя менять после вызова `ResponseWriter.WriteHeader` или `ResponseWriter.Write` + +* Middleware + +.code simpleserver/middleware.go /func RunServerWithMiddleware/,/^}/ + +* Middleware + +.code simpleserver/middleware.go /func UnifiedErrorMiddleware/,/^}/ + +* Graceful shutdown + +.play gracefulshutdown/gracefulshutdown.go /func run()/,/^}/ + +* context + +Контекст - инкапсулированное состояние определенной части приложения. +Позволяет оповещать операции о необходимости завершения и/или передавать контекстозависимые значения между различными частями приложения. +Контексты наследуемы, при отмене закрывается родительский и все дочерние контексты. + + type Context interface { + // Возвращает время, когда операция будет оповещена о необходимости завершения + Deadline() (deadline time.Time, ok bool) + + // Возвращает канал, который будет закрыт при необходимости завершить операцию + // Служит в качестве инструмента оповещения об отмене + Done() <-chan struct{} + + // Если Done не закрыт - возвращает nil. + // Если Done закрыт, Err ошибку с объяснением причины: + // - Canceled - контекст был отменен + // - DeadlineExceeded - наступил дедлайн. + // После возврашения не nil ошибки Err всегда возвращает данную ошибку. + Err() error + + // Позволяет получить произвольный объект из контекста + Value(key interface{}) interface{} + } + +Типы контекстов: +- TODO +- Background +- Cancel +- Deadline + +* Отменяем операции + +.code context/cancelation/cancelation.go /func SimpleCancelation()/,/OMIT/ + +* context в библиотеках Go + +По соглашению `Context` всегда передается первым параметром в функции, обычно именуясь `ctx`. + + database/sql.(*DB).QueryContext(ctx context.Context, query string, args ...interface{}) (*Rows, error) + database/sql.(*DB).ExecContext(ctx context.Context, query string, args ...interface{}) (Result, error) + net/http.NewRequestWithContext(ctx context.Context, method, url string, body io.Reader) (*Request, error) + golang.org/x/sync/errgroup.WithContext(ctx context.Context) (*Group, context.Context) + ... + +Быстрый пример: + + ctx, cancel := context.WithTimeout(context.Background(), 1 * time.Second) + defer cancel() + + req, _ := http.NewRequestWithContext(ctx, "GET", "http://loremipsum.com", nil) + resp, err := http.DefaultClient.Do(req) + if err != nil { + // возможно тут будет DeadlineExceeded + } + +* Контекст в HTTP сервере + +.code context/httpserver/httpserver.go /type ReqTimeContextKey/,/^}/ + +* Контекст в HTTP сервере + +.code context/httpserver/handler.go /type handler/,/^}/ + +* httptest + +* httptest + +Содержит хелперы для удобного написания тестов для HTTP клиентов и серверов. + + // стартует новый локальный HTTP сервер на слуйчаном свободном порту + httptest.NewServer(http.Handler) + // объект, реализующий интерфейс http.ResponseWriter и дающий доступ к результатам ответа + httptest.NewRecorder() + // возвращает объект, готовый к передаче прямо в http.Handler + httptest.NewRequest(method, target string, body io.Reader) *http.Request + +* Пример тестирования клиента + +.code httptest/code.go /^const \(/,/OMIT/ + +* Пример тестирования клиента + +.code httptest/code_test.go /func TestGetReposCount/,/OMIT/ + +* Пример тестирования сервера + +.code context/httpserver/handler_test.go /func TestHandlerServeHTTP/,/^}/ + +* Полезные библиотеки и фреймворки + +Роутеры: + +.link https://github.com/go-chi/chi +.link https://github.com/julienschmidt/httprouter +.link https://github.com/gorilla/mux + +Фреймворки: + +.link https://github.com/labstack/echo +.link https://github.com/gin-gonic/gin +.link https://github.com/gofiber/fiber \ No newline at end of file diff --git a/lectures/06-http/simpleget/simpleget.go b/lectures/06-http/simpleget/simpleget.go new file mode 100644 index 0000000..cab92c8 --- /dev/null +++ b/lectures/06-http/simpleget/simpleget.go @@ -0,0 +1,14 @@ +package main + +import ( + "fmt" + "net/http" +) + +func main() { + resp, err := http.Get("https://golang.org") + if err != nil { + panic(err) + } + fmt.Println(resp.StatusCode) +} diff --git a/lectures/06-http/simpleserver/middleware.go b/lectures/06-http/simpleserver/middleware.go new file mode 100644 index 0000000..68bb419 --- /dev/null +++ b/lectures/06-http/simpleserver/middleware.go @@ -0,0 +1,50 @@ +package simpleserver + +import ( + "errors" + "net/http" +) + +func RunServerWithMiddleware() { + getOnly := func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + next.ServeHTTP(w, r) + }) + } + + handler := func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("pong")) + } + + err := http.ListenAndServe(":8080", getOnly(http.HandlerFunc(handler))) + if err != nil { + panic(err) + } +} + +func UnifiedErrorMiddleware() { + wrapErrorReply := func(h func(w http.ResponseWriter, r *http.Request) error) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := h(w, r); err != nil { + w.WriteHeader(http.StatusBadRequest) + } + }) + } + + handler := func(w http.ResponseWriter, r *http.Request) error { + if r.URL.Query().Get("secret") != "FtP8lu70XjWj8Stt" { + return errors.New("secret mismatch") + } + _, _ = w.Write([]byte("pong")) + return nil + } + + err := http.ListenAndServe(":8080", wrapErrorReply(handler)) + if err != nil { + panic(err) + } +} diff --git a/lectures/06-http/simpleserver/router.go b/lectures/06-http/simpleserver/router.go new file mode 100644 index 0000000..87e9deb --- /dev/null +++ b/lectures/06-http/simpleserver/router.go @@ -0,0 +1,33 @@ +package simpleserver + +import ( + "net/http" +) + +func RunServerWithRouting() { + router := func(w http.ResponseWriter, r *http.Request) { + switch r.RequestURI { + case "/pong": + pongHandler(w, r) + case "/shmong": + shmongHandler(w, r) + default: + w.WriteHeader(404) + } + } + + err := http.ListenAndServe(":8080", http.HandlerFunc(router)) + if err != nil { + panic(err) + } +} + +func pongHandler(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("pong")) +} + +func shmongHandler(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("shmong")) +} + +// OMIT diff --git a/lectures/06-http/simpleserver/simpleserver.go b/lectures/06-http/simpleserver/simpleserver.go new file mode 100644 index 0000000..8a61602 --- /dev/null +++ b/lectures/06-http/simpleserver/simpleserver.go @@ -0,0 +1,27 @@ +package simpleserver + +import ( + "net/http" +) + +func RunServer() { + handler := func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("pong")) + } + + err := http.ListenAndServe(":8080", http.HandlerFunc(handler)) + if err != nil { + panic(err) + } +} + +func RunTLSServer() { + handler := func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("pong")) + } + + err := http.ListenAndServeTLS(":8080", "cert.crt", "private.key", http.HandlerFunc(handler)) + if err != nil { + panic(err) + } +}