Merge branch 'io-lecture' into 'master'

io lecture

See merge request slon/shad-go-private!31

lectures/09-io/bytesbuffer/main.go

@@ -0,0 +1,14 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+)
+
+func main() {
+	var b bytes.Buffer // A Buffer needs no initialization.
+	b.Write([]byte("Hello "))
+	_, _ = fmt.Fprintf(&b, "world!")
+	_, _ = b.WriteTo(os.Stdout)
+}

lectures/09-io/discard/main.go

@@ -0,0 +1,11 @@
+package main
+
+import (
+	"io"
+	"io/ioutil"
+	"strings"
+)
+
+func main() {
+	_, _ = io.Copy(ioutil.Discard, strings.NewReader("nothing of use"))
+}

lectures/09-io/httpchunking/solution1/main.go

@@ -0,0 +1,20 @@
+package main
+
+import (
+	"io"
+	"os"
+	"strings"
+)
+
+func transfer(clientWriter io.Writer, responseBody io.Reader) {
+	_, _ = io.Copy(clientWriter, responseBody)
+}
+
+func parseTrailers(r io.Reader) {
+	_, _ = io.Copy(os.Stdout, r)
+}
+
+func main() {
+	data := "4\r\nWiki\r\n5\r\npedia\r\nE\r\n in\r\n\r\nchunks.\r\n0\r\nDate: Sun, 06 Nov 1994 08:49:37 GMT\r\nContent-MD5: 1B2M2Y8AsgTpgAmY7PhCfg==\r\n\r\n"
+	transfer(os.Stdout, strings.NewReader(data))
+}

lectures/09-io/httpchunking/solution2/main.go

@@ -0,0 +1,25 @@
+package main
+
+import (
+	"io"
+	"net/http/httputil"
+	"os"
+	"strings"
+)
+
+func transfer(clientWriter io.Writer, responseBody io.Reader) {
+	_, _ = io.Copy(
+		clientWriter,
+		httputil.NewChunkedReader(responseBody),
+	)
+	parseTrailers(responseBody)
+}
+
+func parseTrailers(r io.Reader) {
+	_, _ = io.Copy(os.Stdout, r)
+}
+
+func main() {
+	data := "4\r\nWiki\r\n5\r\npedia\r\nE\r\n in\r\n\r\nchunks.\r\n0\r\nDate: Sun, 06 Nov 1994 08:49:37 GMT\r\nContent-MD5: 1B2M2Y8AsgTpgAmY7PhCfg==\r\n\r\n"
+	transfer(os.Stdout, strings.NewReader(data))
+}

lectures/09-io/httpchunking/solution3/main.go

@@ -0,0 +1,31 @@
+package main
+
+import (
+	"crypto/md5"
+	"io"
+	"net/http/httputil"
+	"os"
+	"strings"
+)
+
+func transfer(clientWriter io.Writer, responseBody io.Reader) {
+	digest := md5.New()
+	_, _ = io.Copy(digest,
+		httputil.NewChunkedReader(
+			io.TeeReader(
+				responseBody,
+				clientWriter,
+			),
+		),
+	)
+	parseTrailers(responseBody)
+}
+
+func parseTrailers(r io.Reader) {
+	_, _ = io.Copy(os.Stdout, r)
+}
+
+func main() {
+	data := "4\r\nWiki\r\n5\r\npedia\r\nE\r\n in\r\n\r\nchunks.\r\n0\r\nDate: Sun, 06 Nov 1994 08:49:37 GMT\r\nContent-MD5: 1B2M2Y8AsgTpgAmY7PhCfg==\r\n\r\n"
+	transfer(os.Stdout, strings.NewReader(data))
+}

lectures/09-io/lecture.slide

@@ -0,0 +1,445 @@
+io
+Лекция 9
+
+Арсений Балобанов
+
+* io.Reader & io.Writer
+
+  type Reader interface {
+      Read(p []byte) (n int, err error)
+  }
+
+  type Writer interface {
+      Write(p []byte) (n int, err error)
+  }
+
+*Conceptually:*
+
+- *Reader* has data, and you read it out and make use of that data.
+- You have data and you want to shove it into a *Writer* where something happens to that data.
+
+* io.Reader
+
+  Read(p []byte) (n int, err error)
+
+- Reads up to len(p) bytes into p.
+- Returns the number of bytes read (0 <= n <= len(p)) and any error encountered.
+- If some data is available but not len(p) bytes, Read conventionally returns what is available instead of waiting for more.
+
+* io.Reader
+
+  Read(p []byte) (n int, err error)
+
+- Non-zero number of bytes at the end may result in either err == EOF or err == nil. The next Read should return 0, EOF.
+- Even if Read returns n < len(p), it may use all of p as scratch space during the call (e.g. decompression).
+- Process the n > 0 bytes returned before considering the error err.
+- Implementations must not retain p.
+
+* Reader Variants
+
+  type ReadWriter interface { Reader; Writer }
+  type ReadCloser interface { Reader; Closer }
+  type ReadSeeker interface { Reader; Seeker }
+  type ReadWriteCloser interface { Reader; Writer; Closer }
+  type ReadWriteSeeker interface { Reader; Writer; Seeker }
+
+  type ByteReader interface { ReadByte() (byte, error) }                 // reads single byte
+  type RuneReader interface { ReadRune() (r rune, size int, err error) } // reads single rune
+
+  type LimitedReader struct // limited to N bytes
+  type PipeReader struct    // read half of a pipe
+  type SectionReader struct // read interior section
+
+* SectionReader
+
+.play -edit sectionreader/main.go /^func main/,/^}/
+
+* Reader Composition
+  // LimitReader returns a Reader that reads from r but stops with EOF after n bytes.
+  func LimitReader(r Reader, n int64) Reader
+
+  // MultiReader returns a Reader that's the logical concatenation of the provided input readers.
+  func MultiReader(readers ...Reader) Reader
+
+  // TeeReader returns a Reader that writes to w what it reads from r.
+  func TeeReader(r Reader, w Writer) Reader
+
+* LimitReader
+
+.play -edit limitreader/main.go /^func main/,/^}/
+
+* MultiReader
+
+.play -edit multireader/main.go /^func main/,/^}/
+
+* TeeReader
+
+.play -edit teereader/main.go /^func main/,/^}/
+
+* Example: http chunking
+
+Chunked transfer encoding is a streaming data transfer mechanism.
+
+  4\r\n
+  Wiki\r\n
+  5\r\n
+  pedia\r\n
+  E\r\n
+   in\r\n
+  \r\n
+  chunks.\r\n
+  0\r\n
+  Date: Sun, 06 Nov 1994 08:49:37 GMT\r\n
+  Content-MD5: 1B2M2Y8AsgTpgAmY7PhCfg==\r\n
+  \r\n
+
+Actual body
+
+  Wikipedia in
+
+  chunks.
+
+* Example: http chunking
+
+* Example: http chunking
+
+Problem: proxy a chunked HTTP in a stream.
+
+  func transfer(clientWriter io.Writer, responseBody io.Reader)
+
+- send chunks as is
+- validate MD5
+
+* Example: http chunking
+
+.play -edit httpchunking/solution1/main.go
+
+* Example: http chunking
+
+.play -edit httpchunking/solution2/main.go  /^func transfer/,/^}/
+
+* Example: http chunking
+
+.play -edit httpchunking/solution3/main.go  /^func transfer/,/^}/
+
+* ioutil
+
+Package *io/ioutil* implements some I/O utility functions.
+
+  var Discard io.Writer = devNull(0)
+
+  func NopCloser(r io.Reader) io.ReadCloser
+  func ReadAll(r io.Reader) ([]byte, error)
+  func ReadDir(dirname string) ([]os.FileInfo, error)
+  func ReadFile(filename string) ([]byte, error)
+  func TempDir(dir, pattern string) (name string, err error)
+  func TempFile(dir, pattern string) (f *os.File, err error)
+  func WriteFile(filename string, data []byte, perm os.FileMode) error
+
+* ReadAll
+
+Convenience method for Reader → []byte conversion.
+
+.play -edit readall/main.go /^func main/,/^}/
+
+* ReadAll misuse
+  ReaderAll(Reader) → []byte → Writer
+
+Consider *io.Copy* instead
+
+  io.Copy(dst Writer, src Reader)
+
+* io.Copy
+
+  func Copy(dst Writer, src Reader) (written int64, err error)
+
+- Allocates a 32KB buffer to read from src and then write to dst.
+
+  func CopyBuffer(dst Writer, src Reader, buf []byte) (written int64, err error)
+
+- Reuse your own buffer with CopyBuffer()
+
+Can we avoid using an intermediate buffer entirely?
+
+  type ReaderFrom interface {
+      ReadFrom(r Reader) (n int64, err error)
+  }
+
+  type WriterTo interface {
+      WriteTo(w Writer) (n int64, err error)
+  }
+
+* Example: sendfile
+
+* Example: sendfile
+
+.code -edit sendfile/main.go /^func readFileHandler/,/^}/
+
+* Example: sendfile
+
+  ✗ strace ./sendfile
+  fstat(6, {st_mode=S_IFREG|0644, st_size=1338, ...}) = 0
+  read(6, "{\"id\":\"hello\",\"type"..., 1850) = 1338
+  read(6, "", 512)                        = 0
+  close(6)                                = 0
+  write(4, "HTTP/1.1 200 OK\r\nContent-Disposi"..., 1515) = 1515
+
+* Example: sendfile
+
+.code -edit sendfile/main.go /^func copyHandler/,/^}/
+
+* Example: sendfile
+
+  ✗ strace ./sendfile
+  read(6, "{\"id\":\"hello\",\"type"..., 512) = 512
+  fstat(6, {st_mode=S_IFREG|0644, st_size=1338, ...}) = 0
+  lseek(6, 0, SEEK_SET)                   = 0
+  fstat(6, {st_mode=S_IFREG|0644, st_size=1338, ...}) = 0
+  write(4, "HTTP/1.1 200 OK\r\nContent-Disposi"..., 177) = 177
+  sendfile(4, 6, NULL, 4194304)           = 1338
+  sendfile(4, 6, NULL, 4194304)           = 0
+  close(6)                                = 0
+
+* Example: sendfile
+
+- http.ResponseWriter's is an io.ReaderFrom that uses the implementation of underlying tcp conn.
+
+  // ReadFrom is here to optimize copying from an *os.File regular file
+  // to a *net.TCPConn with sendfile.
+  func (w *response) ReadFrom(src io.Reader) (n int64, err error) {
+
+- net.TCPConn also implements io.ReaderFrom that uses *sendfile* system call.
+
+  func (c *TCPConn) ReadFrom(r io.Reader) (int64, error) {
+
+* ioutil.Discard
+
+  // Discard is an io.Writer on which all Write calls succeed
+  // without doing anything.
+  var Discard io.Writer = devNull(0)
+
+- Implements io.ReaderFrom!
+
+.play -edit discard/main.go
+
+* Reader implementations
+
+Readers are all over the standard library:
+
+- bufio iotest bytes strings crypto debug packet
+- archive/...
+- image/...
+- compress/...
+- encoding/...
+- text/...
+- // and many more...
+
+* io.Writer
+
+  Write(p []byte) (n int, err error)
+
+- Write must return a non-nil error if it returns n < len(p)
+- Write must not modify the slice data, even temporarily
+- Implementations must not retain p
+
+* Writer Variants
+
+  type ReadWriter interface { Reader; Writer }
+  type StringWriter interface { WriteString(s string) (n int, err error) }
+  type ByteWriter interface { WriteByte(c byte) error }
+  type PipeWriter struct // the write half of a pipe
+
+*Composition*
+
+  // MultiWriter creates a writer that duplicates its writes to all the provided writers, similar to the Unix tee(1) command.
+  func MultiWriter(writers ...Writer) Writer
+
+* Writer implementations
+
+- iotest/...
+- archive/...
+- compress/...
+- text/...
+- nex/http/ResponseWriter
+- // and many more
+
+* io.Pipe
+
+.play -edit pipe/main.go /^func main/,/^}/
+
+- synchronous
+- in-memory
+- no internal buffering
+
+* iotest
+
+Package iotest implements Readers and Writers useful mainly for testing.
+
+  // DataErrReader creates a reader that returns (n > 0, EOF) at the end.
+  func DataErrReader(r io.Reader) io.Reader
+
+  // HalfReader returns a Reader that implements Read
+  // by reading half as many requested bytes from r.
+  func HalfReader(r io.Reader) io.Reader
+
+  // OneByteReader returns a Reader that
+  // implements each non-empty Read by reading one byte from r.
+  func OneByteReader(r io.Reader) io.Reader
+
+  // TimeoutReader returns ErrTimeout on the second read with no data.
+  // Subsequent calls to read succeed.
+  func TimeoutReader(r io.Reader) io.Reader
+
+  // TruncateWriter returns a Writer that writes to w but stops silently after n bytes.
+  func TruncateWriter(w io.Writer, n int64) io.Writer
+
+* bufio
+
+Package bufio implements buffered I/O. It wraps an io.Reader or io.Writer objects.
+
+  type Reader
+      func NewReader(rd io.Reader) *Reader
+      func (b *Reader) Discard(n int) (discarded int, err error)
+      func (b *Reader) Peek(n int) ([]byte, error)
+      func (b *Reader) Read(p []byte) (n int, err error)
+      func (b *Reader) ReadByte() (byte, error)
+      func (b *Reader) ReadBytes(delim byte) ([]byte, error)
+      func (b *Reader) ReadLine() (line []byte, isPrefix bool, err error)
+      func (b *Reader) ReadRune() (r rune, size int, err error)
+      func (b *Reader) ReadSlice(delim byte) (line []byte, err error)
+      func (b *Reader) ReadString(delim byte) (string, error)
+
+  type Writer
+      func NewWriter(w io.Writer) *Writer
+      func (b *Writer) Flush() error
+      func (b *Writer) ReadFrom(r io.Reader) (n int64, err error)
+      func (b *Writer) Write(p []byte) (nn int, err error)
+      func (b *Writer) WriteByte(c byte) error
+      func (b *Writer) WriteRune(r rune) (size int, err error)
+      func (b *Writer) WriteString(s string) (int, error)
+
+* bufio.Scanner
+
+Utility type to efficiently read independent lines of text from an io.Reader.
+
+.code -edit scanner/main.go
+
+* bufio read line
+
+There are multiple options to read single line. Which one to use
+*ReadBytes('\n')* or *ReadString('\n')* or *ReadLine* or *Scanner*?
+
+- *ReadBytes* returns the slice together with delimiter
+- *ReadLine* doesn’t handle lines longer than internal buffer (default size 4096)
+- *Scanner* has limited max size of the token (64 * 1024)
+- *ReadLine* needs to be called for the second time to retrieve rest of the stream
+- *ReadBytes* doesn’t have any limit
+- *Scanner* has the simplest API and provides nicest abstraction for common cases
+
+* bufio
+
+- The net/http package already buffers data (using bufio itself) so you don't need this package for that
+- If you are reading a file in one or a few large steps, you probably don't need buffering
+
+* bytes.Buffer
+
+A handy wrapper around byte slice implementing `io.Reader` and `io.Writer`.
+
+Useful when you want to use code that takes an io.Writer, and store the results in memory for the later use.
+
+.play -edit bytesbuffer/main.go
+
+* bytes.Buffer vs strings.Builder
+
+  bytes.Buffer
+      func NewBufferString(s string) *Buffer
+      func (b *Buffer) Bytes() []byte
+      func (b *Buffer) Grow(n int)
+      func (b *Buffer) Read(p []byte) (n int, err error)
+      func (b *Buffer) ReadByte() (byte, error)
+      // other read methods
+      func (b *Buffer) Reset()
+      func (b *Buffer) String() string
+      func (b *Buffer) Write(p []byte) (n int, err error)
+      func (b *Buffer) WriteByte(c byte) error
+      func (b *Buffer) WriteRune(r rune) (n int, err error)
+      func (b *Buffer) WriteString(s string) (n int, err error)
+
+  strings.Builder
+      func (b *Builder) Grow(n int)
+      func (b *Builder) Reset()
+      func (b *Builder) String() string
+      func (b *Builder) Write(p []byte) (int, error)
+      func (b *Builder) WriteByte(c byte) error
+      func (b *Builder) WriteRune(r rune) (int, error)
+      func (b *Builder) WriteString(s string) (int, error)
+
+* bytes.Buffer vs strings.Builder
+
+- *strings.Builder* is immutable and can only grow or reset
+- *bytes.Buffer*'s internal byte slice can escape: (*Buffer).Bytes().
+- strings.Builder.String() does not allocate/copy
+
+  // String returns the accumulated string.
+  func (b *Builder) String() string {
+      return *(*string)(unsafe.Pointer(&b.buf))
+  }
+
+- bytes.Buffer.String() does
+
+  func (b *Buffer) String() string {
+      if b == nil {
+          // Special case, useful in debugging.
+          return "<nil>"
+      }
+      return string(b.buf[b.off:])
+  }
+
+* strings.Builder vs strings.Builder
+
+.play -edit stringsbuilder/main.go
+
+* bytes.Buffer vs strings.Builder
+
+- *strings.Builder* has a copy check
+
+  var b1 strings.Builder
+  b1.WriteString("ABC")
+  b2 := b1
+  b2.WriteString("DEF")
+  // illegal use of non-zero Builder copied by value
+
+- Use pointer to share.
+
+* *os.File
+
+- portable
+- implements io.Reader, and io.Writer which stream bytes to or from a file on disk
+- useful if you don't want to read the whole file into memory
+- has no internal buffers
+
+*ioutil.ReadFile* reads an entire file into memory (as a []byte) in a single call
+
+- allocates a byte slice of the correct size (no need to Read + append in a loop)
+- closes the file
+- returns the first error that prevented it from working
+
+There are also
+
+  func WriteFile(filename string, data []byte, perm os.FileMode) error
+  func TempFile(dir, pattern string) (f *os.File, err error)
+
+* Summary
+
+- *io* defines interfaces that handle streams of bytes (Reader, Writer, etc...) as well as functions that work generically with types implement these interfaces (e.g. io.Copy)
+- *io/ioutil* provides helper functions for some non-trivial file and io tasks
+- *testing/iotest* implements Readers and Writers useful mainly for testing
+- *bufio* provides buffering wrapper for io.Reader and io.Writer that can improve efficiency
+- *bytes* provides helper functions and types for interacting with byte slices
+- ***os.File* implements both io.Reader and io.Writer (among others)
+
+* Links
+
+.link https://medium.com/go-walkthrough/go-walkthrough-io-package-8ac5e95a9fbd io walkthrough
+.link https://medium.com/golangspec/introduction-to-bufio-package-in-golang-ad7d1877f762 bufio walkthrough
+.link https://www.youtube.com/watch?v=kTAsciVuZLQ advanced patterns with io.ReadWriter

lectures/09-io/limitreader/main.go

@@ -0,0 +1,17 @@
+package main
+
+import (
+	"io"
+	"log"
+	"os"
+	"strings"
+)
+
+func main() {
+	r := strings.NewReader("some io.Reader stream to be read\n")
+	lr := io.LimitReader(r, 4)
+
+	if _, err := io.Copy(os.Stdout, lr); err != nil {
+		log.Fatal(err)
+	}
+}

lectures/09-io/multireader/main.go

@@ -0,0 +1,19 @@
+package main
+
+import (
+	"io"
+	"log"
+	"os"
+	"strings"
+)
+
+func main() {
+	r1 := strings.NewReader("first reader ")
+	r2 := strings.NewReader("second reader ")
+	r3 := strings.NewReader("third reader\n")
+	r := io.MultiReader(r1, r2, r3)
+
+	if _, err := io.Copy(os.Stdout, r); err != nil {
+		log.Fatal(err)
+	}
+}

lectures/09-io/pipe/main.go

@@ -0,0 +1,20 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+)
+
+func main() {
+	r, w := io.Pipe()
+
+	go func() {
+		_, _ = fmt.Fprint(w, "some text to be read\n")
+		_ = w.Close()
+	}()
+
+	buf := new(bytes.Buffer)
+	_, _ = buf.ReadFrom(r)
+	fmt.Print(buf.String())
+}

lectures/09-io/readall/main.go

@@ -0,0 +1,20 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"strings"
+)
+
+func main() {
+	r := strings.NewReader("Go is a general-purpose language " +
+		"designed with systems programming in mind.")
+
+	b, err := ioutil.ReadAll(r)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	fmt.Printf("%s", b)
+}

lectures/09-io/scanner/main.go

@@ -0,0 +1,17 @@
+package main
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+)
+
+func main() {
+	scanner := bufio.NewScanner(os.Stdin)
+	for scanner.Scan() {
+		fmt.Println(scanner.Text()) // Println will add back the final '\n'
+	}
+	if err := scanner.Err(); err != nil {
+		_, _ = fmt.Fprintln(os.Stderr, "reading standard input:", err)
+	}
+}

lectures/09-io/sectionreader/main.go

@@ -0,0 +1,17 @@
+package main
+
+import (
+	"io"
+	"log"
+	"os"
+	"strings"
+)
+
+func main() {
+	r := strings.NewReader("some io.Reader stream to be read\n")
+	s := io.NewSectionReader(r, 5, 17)
+
+	if _, err := io.Copy(os.Stdout, s); err != nil {
+		log.Fatal(err)
+	}
+}

lectures/09-io/sendfile/main.go

@@ -0,0 +1,58 @@
+package main
+
+import (
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"strconv"
+)
+
+func main() {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/readFile", readFileHandler)
+	mux.HandleFunc("/copy", copyHandler)
+	log.Fatal(http.ListenAndServe(":8080", mux))
+}
+
+func readFileHandler(w http.ResponseWriter, r *http.Request) {
+	filename := r.URL.Query().Get("file")
+	data, _ := ioutil.ReadFile(filename)
+
+	// Infer the Content-Type of the file.
+	contentType := http.DetectContentType(data[:512])
+
+	// Get the file size.
+	fileSize := strconv.FormatInt(int64(len(data)), 10)
+
+	// Send the headers.
+	w.Header().Set("Content-Disposition", "attachment; filename="+filename)
+	w.Header().Set("Content-Type", contentType)
+	w.Header().Set("Content-Length", fileSize)
+
+	_, _ = w.Write(data)
+}
+
+func copyHandler(w http.ResponseWriter, r *http.Request) {
+	filename := r.URL.Query().Get("file")
+	f, _ := os.Open(filename)
+	defer func() { _ = f.Close() }()
+
+	// Infer the Content-Type of the file.
+	filePrefix := make([]byte, 512)
+	_, _ = f.Read(filePrefix)
+	contentType := http.DetectContentType(filePrefix)
+
+	// Get the file size.
+	fstat, _ := f.Stat()
+	fileSize := strconv.FormatInt(fstat.Size(), 10)
+
+	// Send the headers.
+	w.Header().Set("Content-Disposition", "attachment; filename="+filename)
+	w.Header().Set("Content-Type", contentType)
+	w.Header().Set("Content-Length", fileSize)
+
+	_, _ = f.Seek(0, 0)
+	_, _ = io.Copy(w, f)
+}

lectures/09-io/stringsbuilder/main.go

@@ -0,0 +1,18 @@
+package main
+
+import (
+	"fmt"
+	"strings"
+)
+
+func concat(x, y string) string {
+	var builder strings.Builder
+	builder.Grow(len(x) + len(y)) // only this line allocates
+	builder.WriteString(x)
+	builder.WriteString(y)
+	return builder.String()
+}
+
+func main() {
+	fmt.Println(concat("hello ", "world"))
+}

lectures/09-io/teereader/main.go

@@ -0,0 +1,28 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"strings"
+)
+
+func main() {
+	r := strings.NewReader("some io.Reader stream to be read\n")
+	var buf bytes.Buffer
+	tee := io.TeeReader(r, &buf)
+
+	printall := func(r io.Reader) {
+		b, err := ioutil.ReadAll(r)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		fmt.Printf("%s", b)
+	}
+
+	printall(tee)
+	printall(&buf)
+}