From 51a73f98b2dce725215af26a14b77e537d7b68db Mon Sep 17 00:00:00 2001 From: Arseny Balobanov Date: Thu, 23 Apr 2020 04:40:13 +0300 Subject: [PATCH 1/2] adding io lecture; first commit --- lectures/09-io/bytesbuffer/main.go | 14 ++ lectures/09-io/discard/main.go | 11 + lectures/09-io/lecture.slide | 298 +++++++++++++++++++++++++++ lectures/09-io/limitreader/main.go | 17 ++ lectures/09-io/multireader/main.go | 19 ++ lectures/09-io/pipe/main.go | 20 ++ lectures/09-io/readall/main.go | 20 ++ lectures/09-io/scanner/main.go | 17 ++ lectures/09-io/sectionreader/main.go | 17 ++ lectures/09-io/sendfile/main.go | 58 ++++++ lectures/09-io/teereader/main.go | 29 +++ 11 files changed, 520 insertions(+) create mode 100644 lectures/09-io/bytesbuffer/main.go create mode 100644 lectures/09-io/discard/main.go create mode 100644 lectures/09-io/lecture.slide create mode 100644 lectures/09-io/limitreader/main.go create mode 100644 lectures/09-io/multireader/main.go create mode 100644 lectures/09-io/pipe/main.go create mode 100644 lectures/09-io/readall/main.go create mode 100644 lectures/09-io/scanner/main.go create mode 100644 lectures/09-io/sectionreader/main.go create mode 100644 lectures/09-io/sendfile/main.go create mode 100644 lectures/09-io/teereader/main.go diff --git a/lectures/09-io/bytesbuffer/main.go b/lectures/09-io/bytesbuffer/main.go new file mode 100644 index 0000000..a33f490 --- /dev/null +++ b/lectures/09-io/bytesbuffer/main.go @@ -0,0 +1,14 @@ +package main + +import ( + "bytes" + "fmt" + "os" +) + +func main() { + var b bytes.Buffer // A Buffer needs no initialization. + b.Write([]byte("Hello ")) + _, _ = fmt.Fprintf(&b, "world!") + _, _ = b.WriteTo(os.Stdout) +} diff --git a/lectures/09-io/discard/main.go b/lectures/09-io/discard/main.go new file mode 100644 index 0000000..ab3cf9c --- /dev/null +++ b/lectures/09-io/discard/main.go @@ -0,0 +1,11 @@ +package main + +import ( + "io" + "io/ioutil" + "strings" +) + +func main() { + _, _ = io.Copy(ioutil.Discard, strings.NewReader("nothing of use")) +} diff --git a/lectures/09-io/lecture.slide b/lectures/09-io/lecture.slide new file mode 100644 index 0000000..15e3dcf --- /dev/null +++ b/lectures/09-io/lecture.slide @@ -0,0 +1,298 @@ +io +Лекция 9 + +Someone Someone + +* io.Reader & io.Writer + + type Reader interface { + Read(p []byte) (n int, err error) + } + + type Writer interface { + Write(p []byte) (n int, err error) + } + +*Conceptually:* + +- *Reader* has data, and you read it out and make use of that data. +- You have data and you want to shove it into a *Writer* where something happens to that data. + +* io.Reader + + Read(p []byte) (n int, err error) + +- Reads up to len(p) bytes into p. +- Returns the number of bytes read (0 <= n <= len(p)) and any error encountered. +- If some data is available but not len(p) bytes, Read conventionally returns what is available instead of waiting for more. + +* io.Reader + + Read(p []byte) (n int, err error) + +- Non-zero number of bytes at the end may result in either err == EOF or err == nil. The next Read should return 0, EOF. +- Even if Read returns n < len(p), it may use all of p as scratch space during the call (e.g. decompression). +- Process the n > 0 bytes returned before considering the error err. +- Implementations must not retain p. + +* Reader Variants + + type ReadWriter interface { Reader; Writer } + type ReadCloser interface { Reader; Closer } + type ReadSeeker interface { Reader; Seeker } + type ReadWriteCloser interface { Reader; Writer; Closer } + type ReadWriteSeeker interface { Reader; Writer; Seeker } + + type ByteReader interface { ReadByte() (byte, error) } // reads single byte + type RuneReader interface { ReadRune() (r rune, size int, err error) } // reads single rune + + type LimitedReader struct // limited to N bytes + type PipeReader struct // read half of a pipe + type SectionReader struct // read interior section + +* SectionReader + +.play -edit sectionreader/main.go /^func main/,/^}/ + +* Reader Composition + // LimitReader returns a Reader that reads from r but stops with EOF after n bytes. + func LimitReader(r Reader, n int64) Reader + + // MultiReader returns a Reader that's the logical concatenation of the provided input readers. + func MultiReader(readers ...Reader) Reader + + // TeeReader returns a Reader that writes to w what it reads from r. + func TeeReader(r Reader, w Writer) Reader + +* LimitReader + +.play -edit limitreader/main.go /^func main/,/^}/ + +* MultiReader + +.play -edit multireader/main.go /^func main/,/^}/ + +* TeeReader + +.play -edit teereader/main.go /^func main/,/^}/ + +* ioutil + +Package *io/ioutil* implements some I/O utility functions. + + var Discard io.Writer = devNull(0) + + func NopCloser(r io.Reader) io.ReadCloser + func ReadAll(r io.Reader) ([]byte, error) + func ReadDir(dirname string) ([]os.FileInfo, error) + func ReadFile(filename string) ([]byte, error) + func TempDir(dir, pattern string) (name string, err error) + func TempFile(dir, pattern string) (f *os.File, err error) + func WriteFile(filename string, data []byte, perm os.FileMode) error + +* ReadAll + +Convenience method for Reader → []byte conversion. + +.play -edit readall/main.go /^func main/,/^}/ + +* ReadAll misuse + ReaderAll(Reader) → []byte → Writer + +Consider *io.Copy* instead + + io.Copy(dst Writer, src Reader) + +* io.Copy + + func Copy(dst Writer, src Reader) (written int64, err error) + +- Allocates a 32KB buffer to read from src and then write to dst. + + func CopyBuffer(dst Writer, src Reader, buf []byte) (written int64, err error) + +- Reuse your own buffer by with CopyBuffer() + +Can we avoid using an intermediate buffer entirely? + + type ReaderFrom interface { + ReadFrom(r Reader) (n int64, err error) + } + + type WriterTo interface { + WriteTo(w Writer) (n int64, err error) + } + +* Example: sendfile + +.code -edit sendfile/main.go /^func readFileHandler/,/^}/ + +* Example: sendfile + + ✗ strace ./sendfile + fstat(6, {st_mode=S_IFREG|0644, st_size=1338, ...}) = 0 + read(6, "{\"id\":\"hello\",\"type"..., 1850) = 1338 + read(6, "", 512) = 0 + close(6) = 0 + write(4, "HTTP/1.1 200 OK\r\nContent-Disposi"..., 1515) = 1515 + +* Example: sendfile + +.code -edit sendfile/main.go /^func copyHandler/,/^}/ + +* Example: sendfile + + ✗ strace ./sendfile + read(6, "{\"id\":\"hello\",\"type"..., 512) = 512 + fstat(6, {st_mode=S_IFREG|0644, st_size=1338, ...}) = 0 + lseek(6, 0, SEEK_SET) = 0 + fstat(6, {st_mode=S_IFREG|0644, st_size=1338, ...}) = 0 + write(4, "HTTP/1.1 200 OK\r\nContent-Disposi"..., 177) = 177 + sendfile(4, 6, NULL, 4194304) = 1338 + sendfile(4, 6, NULL, 4194304) = 0 + close(6) = 0 + +* Example: sendfile + +- http.ResponseWriter's is a io.ReaderFrom that uses the implementation of underlying tcp conn. + + // ReadFrom is here to optimize copying from an *os.File regular file + // to a *net.TCPConn with sendfile. + func (w *response) ReadFrom(src io.Reader) (n int64, err error) { + +- net.TCPConn also implements io.ReaderFrom that uses *sendfile* system call. + + // ReadFrom implements the io.ReaderFrom ReadFrom method. + func (c *TCPConn) ReadFrom(r io.Reader) (int64, error) { + +* ioutil.Discard + + // Discard is an io.Writer on which all Write calls succeed + // without doing anything. + var Discard io.Writer = devNull(0) + +- Implements io.ReaderFrom! + +.play -edit discard/main.go + +* Reader implementations + +Readers are all over the standard library: + +- bufio iotest bytes strings crypto debug packet +- archive/... +- image/... +- compress/... +- encoding/... +- text/... +- // and many more... + +* io.Writer + + Write(p []byte) (n int, err error) + +- Write must return a non-nil error if it returns n < len(p) +- Write must not modify the slice data, even temporarily +- Implementations must not retain p + +* Writer Variants + + type ReadWriter interface { Reader; Writer } + type StringWriter interface { WriteString(s string) (n int, err error) } + type ByteWriter interface { WriteByte(c byte) error } + type PipeWriter struct // the write half of a pipe + +*Composition* + + // MultiWriter creates a writer that duplicates its writes to all the provided writers, similar to the Unix tee(1) command. + func MultiWriter(writers ...Writer) Writer + +* Writer implementations + +- iotest/... +- archive/... +- compress/... +- text/... +- nex/http/ResponseWriter +- // and many more + +* io.Pipe + +.play -edit pipe/main.go /^func main/,/^}/ + +- synchronous +- in-memory +- no internal buffering + +* iotest + +Package iotest implements Readers and Writers useful mainly for testing. + + // DataErrReader creates a reader that returns (n > 0, EOF) at the end. + func DataErrReader(r io.Reader) io.Reader + + // HalfReader returns a Reader that implements Read + // by reading half as many requested bytes from r. + func HalfReader(r io.Reader) io.Reader + + // OneByteReader returns a Reader that + // implements each non-empty Read by reading one byte from r. + func OneByteReader(r io.Reader) io.Reader + + // TimeoutReader returns ErrTimeout on the second read with no data. + // Subsequent calls to read succeed. + func TimeoutReader(r io.Reader) io.Reader + + // TruncateWriter returns a Writer that writes to w but stops silently after n bytes. + func TruncateWriter(w io.Writer, n int64) io.Writer + +* bufio + +Package bufio implements buffered I/O. It wraps an io.Reader or io.Writer objects. + + type Reader + func NewReader(rd io.Reader) *Reader + func (b *Reader) Discard(n int) (discarded int, err error) + func (b *Reader) Peek(n int) ([]byte, error) + func (b *Reader) Read(p []byte) (n int, err error) + func (b *Reader) ReadByte() (byte, error) + func (b *Reader) ReadBytes(delim byte) ([]byte, error) + func (b *Reader) ReadLine() (line []byte, isPrefix bool, err error) + func (b *Reader) ReadRune() (r rune, size int, err error) + func (b *Reader) ReadSlice(delim byte) (line []byte, err error) + func (b *Reader) ReadString(delim byte) (string, error) + + type Writer + func NewWriter(w io.Writer) *Writer + func (b *Writer) Flush() error + func (b *Writer) ReadFrom(r io.Reader) (n int64, err error) + func (b *Writer) Write(p []byte) (nn int, err error) + func (b *Writer) WriteByte(c byte) error + func (b *Writer) WriteRune(r rune) (size int, err error) + func (b *Writer) WriteString(s string) (int, error) + +* bufio.Scanner + +Utility type to efficiently read independent lines of text from an io.Reader. + +.code -edit scanner/main.go + +* bytes.Buffer + +A handy wrapper around byte slice implementing `io.Reader` and `io.Writer`. + +Useful when you want to use code that takes an io.Writer, and store the results in memory for the later use. + +.play -edit bytesbuffer/main.go + +* *os.File + +- portable +- implements io.Reader, and io.Writer which stream bytes to or from a file on disk +- useful if you don't want to read the whole file into memory + +*ioutil.ReadFile* reads an entire file into memory (as a []byte) in a single call + +- allocates a byte slice of the correct size (no need to Read + append in a loop) +- closes the file +- returns the first error that prevented it from working diff --git a/lectures/09-io/limitreader/main.go b/lectures/09-io/limitreader/main.go new file mode 100644 index 0000000..4903cdc --- /dev/null +++ b/lectures/09-io/limitreader/main.go @@ -0,0 +1,17 @@ +package main + +import ( + "io" + "log" + "os" + "strings" +) + +func main() { + r := strings.NewReader("some io.Reader stream to be read\n") + lr := io.LimitReader(r, 4) + + if _, err := io.Copy(os.Stdout, lr); err != nil { + log.Fatal(err) + } +} diff --git a/lectures/09-io/multireader/main.go b/lectures/09-io/multireader/main.go new file mode 100644 index 0000000..8281004 --- /dev/null +++ b/lectures/09-io/multireader/main.go @@ -0,0 +1,19 @@ +package main + +import ( + "io" + "log" + "os" + "strings" +) + +func main() { + r1 := strings.NewReader("first reader ") + r2 := strings.NewReader("second reader ") + r3 := strings.NewReader("third reader\n") + r := io.MultiReader(r1, r2, r3) + + if _, err := io.Copy(os.Stdout, r); err != nil { + log.Fatal(err) + } +} diff --git a/lectures/09-io/pipe/main.go b/lectures/09-io/pipe/main.go new file mode 100644 index 0000000..0e91b8c --- /dev/null +++ b/lectures/09-io/pipe/main.go @@ -0,0 +1,20 @@ +package main + +import ( + "bytes" + "fmt" + "io" +) + +func main() { + r, w := io.Pipe() + + go func() { + _, _ = fmt.Fprint(w, "some text to be read\n") + _ = w.Close() + }() + + buf := new(bytes.Buffer) + _, _ = buf.ReadFrom(r) + fmt.Print(buf.String()) +} diff --git a/lectures/09-io/readall/main.go b/lectures/09-io/readall/main.go new file mode 100644 index 0000000..6e10263 --- /dev/null +++ b/lectures/09-io/readall/main.go @@ -0,0 +1,20 @@ +package main + +import ( + "fmt" + "io/ioutil" + "log" + "strings" +) + +func main() { + r := strings.NewReader("Go is a general-purpose language " + + "designed with systems programming in mind.") + + b, err := ioutil.ReadAll(r) + if err != nil { + log.Fatal(err) + } + + fmt.Printf("%s", b) +} diff --git a/lectures/09-io/scanner/main.go b/lectures/09-io/scanner/main.go new file mode 100644 index 0000000..eda4979 --- /dev/null +++ b/lectures/09-io/scanner/main.go @@ -0,0 +1,17 @@ +package main + +import ( + "bufio" + "fmt" + "os" +) + +func main() { + scanner := bufio.NewScanner(os.Stdin) + for scanner.Scan() { + fmt.Println(scanner.Text()) // Println will add back the final '\n' + } + if err := scanner.Err(); err != nil { + _, _ = fmt.Fprintln(os.Stderr, "reading standard input:", err) + } +} diff --git a/lectures/09-io/sectionreader/main.go b/lectures/09-io/sectionreader/main.go new file mode 100644 index 0000000..f615ca2 --- /dev/null +++ b/lectures/09-io/sectionreader/main.go @@ -0,0 +1,17 @@ +package main + +import ( + "io" + "log" + "os" + "strings" +) + +func main() { + r := strings.NewReader("some io.Reader stream to be read\n") + s := io.NewSectionReader(r, 5, 17) + + if _, err := io.Copy(os.Stdout, s); err != nil { + log.Fatal(err) + } +} diff --git a/lectures/09-io/sendfile/main.go b/lectures/09-io/sendfile/main.go new file mode 100644 index 0000000..f2608ec --- /dev/null +++ b/lectures/09-io/sendfile/main.go @@ -0,0 +1,58 @@ +package main + +import ( + "io" + "io/ioutil" + "log" + "net/http" + "os" + "strconv" +) + +func main() { + mux := http.NewServeMux() + mux.HandleFunc("/readFile", readFileHandler) + mux.HandleFunc("/copy", copyHandler) + log.Fatal(http.ListenAndServe(":8080", mux)) +} + +func readFileHandler(w http.ResponseWriter, r *http.Request) { + filename := r.URL.Query().Get("file") + data, _ := ioutil.ReadFile(filename) + + // Infer the Content-Type of the file. + contentType := http.DetectContentType(data[:512]) + + // Get the file size. + fileSize := strconv.FormatInt(int64(len(data)), 10) + + // Send the headers. + w.Header().Set("Content-Disposition", "attachment; filename="+filename) + w.Header().Set("Content-Type", contentType) + w.Header().Set("Content-Length", fileSize) + + _, _ = w.Write(data) +} + +func copyHandler(w http.ResponseWriter, r *http.Request) { + filename := r.URL.Query().Get("file") + f, _ := os.Open(filename) + defer func() { _ = f.Close() }() + + // Infer the Content-Type of the file. + filePrefix := make([]byte, 512) + _, _ = f.Read(filePrefix) + contentType := http.DetectContentType(filePrefix) + + // Get the file size. + fstat, _ := f.Stat() + fileSize := strconv.FormatInt(fstat.Size(), 10) + + // Send the headers. + w.Header().Set("Content-Disposition", "attachment; filename="+filename) + w.Header().Set("Content-Type", contentType) + w.Header().Set("Content-Length", fileSize) + + _, _ = f.Seek(0, 0) + _, _ = io.Copy(w, f) +} diff --git a/lectures/09-io/teereader/main.go b/lectures/09-io/teereader/main.go new file mode 100644 index 0000000..cd9efdc --- /dev/null +++ b/lectures/09-io/teereader/main.go @@ -0,0 +1,29 @@ +package main + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "log" + "strings" +) + +func main() { + r := strings.NewReader("some io.Reader stream to be read\n") + var buf bytes.Buffer + tee := io.TeeReader(r, &buf) + + printall := func(r io.Reader) { + b, err := ioutil.ReadAll(r) + if err != nil { + log.Fatal(err) + } + + fmt.Printf("%s", b) + } + + printall(tee) + printall(&buf) + +} From daa4051650c3c14bd71da5dc75c7f0d523563452 Mon Sep 17 00:00:00 2001 From: Arseny Balobanov Date: Thu, 23 Apr 2020 17:39:10 +0300 Subject: [PATCH 2/2] Add strings.Builder vs bytes.Buffer; different options for line reading; http chunking; --- lectures/09-io/httpchunking/solution1/main.go | 20 +++ lectures/09-io/httpchunking/solution2/main.go | 25 +++ lectures/09-io/httpchunking/solution3/main.go | 31 ++++ lectures/09-io/lecture.slide | 155 +++++++++++++++++- lectures/09-io/stringsbuilder/main.go | 18 ++ lectures/09-io/teereader/main.go | 1 - 6 files changed, 245 insertions(+), 5 deletions(-) create mode 100644 lectures/09-io/httpchunking/solution1/main.go create mode 100644 lectures/09-io/httpchunking/solution2/main.go create mode 100644 lectures/09-io/httpchunking/solution3/main.go create mode 100644 lectures/09-io/stringsbuilder/main.go diff --git a/lectures/09-io/httpchunking/solution1/main.go b/lectures/09-io/httpchunking/solution1/main.go new file mode 100644 index 0000000..9c0dacb --- /dev/null +++ b/lectures/09-io/httpchunking/solution1/main.go @@ -0,0 +1,20 @@ +package main + +import ( + "io" + "os" + "strings" +) + +func transfer(clientWriter io.Writer, responseBody io.Reader) { + _, _ = io.Copy(clientWriter, responseBody) +} + +func parseTrailers(r io.Reader) { + _, _ = io.Copy(os.Stdout, r) +} + +func main() { + data := "4\r\nWiki\r\n5\r\npedia\r\nE\r\n in\r\n\r\nchunks.\r\n0\r\nDate: Sun, 06 Nov 1994 08:49:37 GMT\r\nContent-MD5: 1B2M2Y8AsgTpgAmY7PhCfg==\r\n\r\n" + transfer(os.Stdout, strings.NewReader(data)) +} diff --git a/lectures/09-io/httpchunking/solution2/main.go b/lectures/09-io/httpchunking/solution2/main.go new file mode 100644 index 0000000..567f979 --- /dev/null +++ b/lectures/09-io/httpchunking/solution2/main.go @@ -0,0 +1,25 @@ +package main + +import ( + "io" + "net/http/httputil" + "os" + "strings" +) + +func transfer(clientWriter io.Writer, responseBody io.Reader) { + _, _ = io.Copy( + clientWriter, + httputil.NewChunkedReader(responseBody), + ) + parseTrailers(responseBody) +} + +func parseTrailers(r io.Reader) { + _, _ = io.Copy(os.Stdout, r) +} + +func main() { + data := "4\r\nWiki\r\n5\r\npedia\r\nE\r\n in\r\n\r\nchunks.\r\n0\r\nDate: Sun, 06 Nov 1994 08:49:37 GMT\r\nContent-MD5: 1B2M2Y8AsgTpgAmY7PhCfg==\r\n\r\n" + transfer(os.Stdout, strings.NewReader(data)) +} diff --git a/lectures/09-io/httpchunking/solution3/main.go b/lectures/09-io/httpchunking/solution3/main.go new file mode 100644 index 0000000..3d9dadf --- /dev/null +++ b/lectures/09-io/httpchunking/solution3/main.go @@ -0,0 +1,31 @@ +package main + +import ( + "crypto/md5" + "io" + "net/http/httputil" + "os" + "strings" +) + +func transfer(clientWriter io.Writer, responseBody io.Reader) { + digest := md5.New() + _, _ = io.Copy(digest, + httputil.NewChunkedReader( + io.TeeReader( + responseBody, + clientWriter, + ), + ), + ) + parseTrailers(responseBody) +} + +func parseTrailers(r io.Reader) { + _, _ = io.Copy(os.Stdout, r) +} + +func main() { + data := "4\r\nWiki\r\n5\r\npedia\r\nE\r\n in\r\n\r\nchunks.\r\n0\r\nDate: Sun, 06 Nov 1994 08:49:37 GMT\r\nContent-MD5: 1B2M2Y8AsgTpgAmY7PhCfg==\r\n\r\n" + transfer(os.Stdout, strings.NewReader(data)) +} diff --git a/lectures/09-io/lecture.slide b/lectures/09-io/lecture.slide index 15e3dcf..c193975 100644 --- a/lectures/09-io/lecture.slide +++ b/lectures/09-io/lecture.slide @@ -1,7 +1,7 @@ io Лекция 9 -Someone Someone +Арсений Балобанов * io.Reader & io.Writer @@ -76,6 +76,52 @@ Someone Someone .play -edit teereader/main.go /^func main/,/^}/ +* Example: http chunking + +Chunked transfer encoding is a streaming data transfer mechanism. + + 4\r\n + Wiki\r\n + 5\r\n + pedia\r\n + E\r\n + in\r\n + \r\n + chunks.\r\n + 0\r\n + Date: Sun, 06 Nov 1994 08:49:37 GMT\r\n + Content-MD5: 1B2M2Y8AsgTpgAmY7PhCfg==\r\n + \r\n + +Actual body + + Wikipedia in + + chunks. + +* Example: http chunking + +* Example: http chunking + +Problem: proxy a chunked HTTP in a stream. + + func transfer(clientWriter io.Writer, responseBody io.Reader) + +- send chunks as is +- validate MD5 + +* Example: http chunking + +.play -edit httpchunking/solution1/main.go + +* Example: http chunking + +.play -edit httpchunking/solution2/main.go /^func transfer/,/^}/ + +* Example: http chunking + +.play -edit httpchunking/solution3/main.go /^func transfer/,/^}/ + * ioutil Package *io/ioutil* implements some I/O utility functions. @@ -111,7 +157,7 @@ Consider *io.Copy* instead func CopyBuffer(dst Writer, src Reader, buf []byte) (written int64, err error) -- Reuse your own buffer by with CopyBuffer() +- Reuse your own buffer with CopyBuffer() Can we avoid using an intermediate buffer entirely? @@ -125,6 +171,8 @@ Can we avoid using an intermediate buffer entirely? * Example: sendfile +* Example: sendfile + .code -edit sendfile/main.go /^func readFileHandler/,/^}/ * Example: sendfile @@ -154,7 +202,7 @@ Can we avoid using an intermediate buffer entirely? * Example: sendfile -- http.ResponseWriter's is a io.ReaderFrom that uses the implementation of underlying tcp conn. +- http.ResponseWriter's is an io.ReaderFrom that uses the implementation of underlying tcp conn. // ReadFrom is here to optimize copying from an *os.File regular file // to a *net.TCPConn with sendfile. @@ -162,7 +210,6 @@ Can we avoid using an intermediate buffer entirely? - net.TCPConn also implements io.ReaderFrom that uses *sendfile* system call. - // ReadFrom implements the io.ReaderFrom ReadFrom method. func (c *TCPConn) ReadFrom(r io.Reader) (int64, error) { * ioutil.Discard @@ -277,6 +324,23 @@ Utility type to efficiently read independent lines of text from an io.Reader. .code -edit scanner/main.go +* bufio read line + +There are multiple options to read single line. Which one to use +*ReadBytes('\n')* or *ReadString('\n')* or *ReadLine* or *Scanner*? + +- *ReadBytes* returns the slice together with delimiter +- *ReadLine* doesn’t handle lines longer than internal buffer (default size 4096) +- *Scanner* has limited max size of the token (64 * 1024) +- *ReadLine* needs to be called for the second time to retrieve rest of the stream +- *ReadBytes* doesn’t have any limit +- *Scanner* has the simplest API and provides nicest abstraction for common cases + +* bufio + +- The net/http package already buffers data (using bufio itself) so you don't need this package for that +- If you are reading a file in one or a few large steps, you probably don't need buffering + * bytes.Buffer A handy wrapper around byte slice implementing `io.Reader` and `io.Writer`. @@ -285,14 +349,97 @@ Useful when you want to use code that takes an io.Writer, and store the results .play -edit bytesbuffer/main.go +* bytes.Buffer vs strings.Builder + + bytes.Buffer + func NewBufferString(s string) *Buffer + func (b *Buffer) Bytes() []byte + func (b *Buffer) Grow(n int) + func (b *Buffer) Read(p []byte) (n int, err error) + func (b *Buffer) ReadByte() (byte, error) + // other read methods + func (b *Buffer) Reset() + func (b *Buffer) String() string + func (b *Buffer) Write(p []byte) (n int, err error) + func (b *Buffer) WriteByte(c byte) error + func (b *Buffer) WriteRune(r rune) (n int, err error) + func (b *Buffer) WriteString(s string) (n int, err error) + + strings.Builder + func (b *Builder) Grow(n int) + func (b *Builder) Reset() + func (b *Builder) String() string + func (b *Builder) Write(p []byte) (int, error) + func (b *Builder) WriteByte(c byte) error + func (b *Builder) WriteRune(r rune) (int, error) + func (b *Builder) WriteString(s string) (int, error) + +* bytes.Buffer vs strings.Builder + +- *strings.Builder* is immutable and can only grow or reset +- *bytes.Buffer*'s internal byte slice can escape: (*Buffer).Bytes(). +- strings.Builder.String() does not allocate/copy + + // String returns the accumulated string. + func (b *Builder) String() string { + return *(*string)(unsafe.Pointer(&b.buf)) + } + +- bytes.Buffer.String() does + + func (b *Buffer) String() string { + if b == nil { + // Special case, useful in debugging. + return "" + } + return string(b.buf[b.off:]) + } + +* strings.Builder vs strings.Builder + +.play -edit stringsbuilder/main.go + +* bytes.Buffer vs strings.Builder + +- *strings.Builder* has a copy check + + var b1 strings.Builder + b1.WriteString("ABC") + b2 := b1 + b2.WriteString("DEF") + // illegal use of non-zero Builder copied by value + +- Use pointer to share. + * *os.File - portable - implements io.Reader, and io.Writer which stream bytes to or from a file on disk - useful if you don't want to read the whole file into memory +- has no internal buffers *ioutil.ReadFile* reads an entire file into memory (as a []byte) in a single call - allocates a byte slice of the correct size (no need to Read + append in a loop) - closes the file - returns the first error that prevented it from working + +There are also + + func WriteFile(filename string, data []byte, perm os.FileMode) error + func TempFile(dir, pattern string) (f *os.File, err error) + +* Summary + +- *io* defines interfaces that handle streams of bytes (Reader, Writer, etc...) as well as functions that work generically with types implement these interfaces (e.g. io.Copy) +- *io/ioutil* provides helper functions for some non-trivial file and io tasks +- *testing/iotest* implements Readers and Writers useful mainly for testing +- *bufio* provides buffering wrapper for io.Reader and io.Writer that can improve efficiency +- *bytes* provides helper functions and types for interacting with byte slices +- ***os.File* implements both io.Reader and io.Writer (among others) + +* Links + +.link https://medium.com/go-walkthrough/go-walkthrough-io-package-8ac5e95a9fbd io walkthrough +.link https://medium.com/golangspec/introduction-to-bufio-package-in-golang-ad7d1877f762 bufio walkthrough +.link https://www.youtube.com/watch?v=kTAsciVuZLQ advanced patterns with io.ReadWriter diff --git a/lectures/09-io/stringsbuilder/main.go b/lectures/09-io/stringsbuilder/main.go new file mode 100644 index 0000000..a537455 --- /dev/null +++ b/lectures/09-io/stringsbuilder/main.go @@ -0,0 +1,18 @@ +package main + +import ( + "fmt" + "strings" +) + +func concat(x, y string) string { + var builder strings.Builder + builder.Grow(len(x) + len(y)) // only this line allocates + builder.WriteString(x) + builder.WriteString(y) + return builder.String() +} + +func main() { + fmt.Println(concat("hello ", "world")) +} diff --git a/lectures/09-io/teereader/main.go b/lectures/09-io/teereader/main.go index cd9efdc..d6d02b3 100644 --- a/lectures/09-io/teereader/main.go +++ b/lectures/09-io/teereader/main.go @@ -25,5 +25,4 @@ func main() { printall(tee) printall(&buf) - }