diff --git a/lectures/09-io/bytesbuffer/main.go b/lectures/09-io/bytesbuffer/main.go new file mode 100644 index 0000000..a33f490 --- /dev/null +++ b/lectures/09-io/bytesbuffer/main.go @@ -0,0 +1,14 @@ +package main + +import ( + "bytes" + "fmt" + "os" +) + +func main() { + var b bytes.Buffer // A Buffer needs no initialization. + b.Write([]byte("Hello ")) + _, _ = fmt.Fprintf(&b, "world!") + _, _ = b.WriteTo(os.Stdout) +} diff --git a/lectures/09-io/discard/main.go b/lectures/09-io/discard/main.go new file mode 100644 index 0000000..ab3cf9c --- /dev/null +++ b/lectures/09-io/discard/main.go @@ -0,0 +1,11 @@ +package main + +import ( + "io" + "io/ioutil" + "strings" +) + +func main() { + _, _ = io.Copy(ioutil.Discard, strings.NewReader("nothing of use")) +} diff --git a/lectures/09-io/lecture.slide b/lectures/09-io/lecture.slide new file mode 100644 index 0000000..15e3dcf --- /dev/null +++ b/lectures/09-io/lecture.slide @@ -0,0 +1,298 @@ +io +Лекция 9 + +Someone Someone + +* io.Reader & io.Writer + + type Reader interface { + Read(p []byte) (n int, err error) + } + + type Writer interface { + Write(p []byte) (n int, err error) + } + +*Conceptually:* + +- *Reader* has data, and you read it out and make use of that data. +- You have data and you want to shove it into a *Writer* where something happens to that data. + +* io.Reader + + Read(p []byte) (n int, err error) + +- Reads up to len(p) bytes into p. +- Returns the number of bytes read (0 <= n <= len(p)) and any error encountered. +- If some data is available but not len(p) bytes, Read conventionally returns what is available instead of waiting for more. + +* io.Reader + + Read(p []byte) (n int, err error) + +- Non-zero number of bytes at the end may result in either err == EOF or err == nil. The next Read should return 0, EOF. +- Even if Read returns n < len(p), it may use all of p as scratch space during the call (e.g. decompression). +- Process the n > 0 bytes returned before considering the error err. +- Implementations must not retain p. + +* Reader Variants + + type ReadWriter interface { Reader; Writer } + type ReadCloser interface { Reader; Closer } + type ReadSeeker interface { Reader; Seeker } + type ReadWriteCloser interface { Reader; Writer; Closer } + type ReadWriteSeeker interface { Reader; Writer; Seeker } + + type ByteReader interface { ReadByte() (byte, error) } // reads single byte + type RuneReader interface { ReadRune() (r rune, size int, err error) } // reads single rune + + type LimitedReader struct // limited to N bytes + type PipeReader struct // read half of a pipe + type SectionReader struct // read interior section + +* SectionReader + +.play -edit sectionreader/main.go /^func main/,/^}/ + +* Reader Composition + // LimitReader returns a Reader that reads from r but stops with EOF after n bytes. + func LimitReader(r Reader, n int64) Reader + + // MultiReader returns a Reader that's the logical concatenation of the provided input readers. + func MultiReader(readers ...Reader) Reader + + // TeeReader returns a Reader that writes to w what it reads from r. + func TeeReader(r Reader, w Writer) Reader + +* LimitReader + +.play -edit limitreader/main.go /^func main/,/^}/ + +* MultiReader + +.play -edit multireader/main.go /^func main/,/^}/ + +* TeeReader + +.play -edit teereader/main.go /^func main/,/^}/ + +* ioutil + +Package *io/ioutil* implements some I/O utility functions. + + var Discard io.Writer = devNull(0) + + func NopCloser(r io.Reader) io.ReadCloser + func ReadAll(r io.Reader) ([]byte, error) + func ReadDir(dirname string) ([]os.FileInfo, error) + func ReadFile(filename string) ([]byte, error) + func TempDir(dir, pattern string) (name string, err error) + func TempFile(dir, pattern string) (f *os.File, err error) + func WriteFile(filename string, data []byte, perm os.FileMode) error + +* ReadAll + +Convenience method for Reader → []byte conversion. + +.play -edit readall/main.go /^func main/,/^}/ + +* ReadAll misuse + ReaderAll(Reader) → []byte → Writer + +Consider *io.Copy* instead + + io.Copy(dst Writer, src Reader) + +* io.Copy + + func Copy(dst Writer, src Reader) (written int64, err error) + +- Allocates a 32KB buffer to read from src and then write to dst. + + func CopyBuffer(dst Writer, src Reader, buf []byte) (written int64, err error) + +- Reuse your own buffer by with CopyBuffer() + +Can we avoid using an intermediate buffer entirely? + + type ReaderFrom interface { + ReadFrom(r Reader) (n int64, err error) + } + + type WriterTo interface { + WriteTo(w Writer) (n int64, err error) + } + +* Example: sendfile + +.code -edit sendfile/main.go /^func readFileHandler/,/^}/ + +* Example: sendfile + + ✗ strace ./sendfile + fstat(6, {st_mode=S_IFREG|0644, st_size=1338, ...}) = 0 + read(6, "{\"id\":\"hello\",\"type"..., 1850) = 1338 + read(6, "", 512) = 0 + close(6) = 0 + write(4, "HTTP/1.1 200 OK\r\nContent-Disposi"..., 1515) = 1515 + +* Example: sendfile + +.code -edit sendfile/main.go /^func copyHandler/,/^}/ + +* Example: sendfile + + ✗ strace ./sendfile + read(6, "{\"id\":\"hello\",\"type"..., 512) = 512 + fstat(6, {st_mode=S_IFREG|0644, st_size=1338, ...}) = 0 + lseek(6, 0, SEEK_SET) = 0 + fstat(6, {st_mode=S_IFREG|0644, st_size=1338, ...}) = 0 + write(4, "HTTP/1.1 200 OK\r\nContent-Disposi"..., 177) = 177 + sendfile(4, 6, NULL, 4194304) = 1338 + sendfile(4, 6, NULL, 4194304) = 0 + close(6) = 0 + +* Example: sendfile + +- http.ResponseWriter's is a io.ReaderFrom that uses the implementation of underlying tcp conn. + + // ReadFrom is here to optimize copying from an *os.File regular file + // to a *net.TCPConn with sendfile. + func (w *response) ReadFrom(src io.Reader) (n int64, err error) { + +- net.TCPConn also implements io.ReaderFrom that uses *sendfile* system call. + + // ReadFrom implements the io.ReaderFrom ReadFrom method. + func (c *TCPConn) ReadFrom(r io.Reader) (int64, error) { + +* ioutil.Discard + + // Discard is an io.Writer on which all Write calls succeed + // without doing anything. + var Discard io.Writer = devNull(0) + +- Implements io.ReaderFrom! + +.play -edit discard/main.go + +* Reader implementations + +Readers are all over the standard library: + +- bufio iotest bytes strings crypto debug packet +- archive/... +- image/... +- compress/... +- encoding/... +- text/... +- // and many more... + +* io.Writer + + Write(p []byte) (n int, err error) + +- Write must return a non-nil error if it returns n < len(p) +- Write must not modify the slice data, even temporarily +- Implementations must not retain p + +* Writer Variants + + type ReadWriter interface { Reader; Writer } + type StringWriter interface { WriteString(s string) (n int, err error) } + type ByteWriter interface { WriteByte(c byte) error } + type PipeWriter struct // the write half of a pipe + +*Composition* + + // MultiWriter creates a writer that duplicates its writes to all the provided writers, similar to the Unix tee(1) command. + func MultiWriter(writers ...Writer) Writer + +* Writer implementations + +- iotest/... +- archive/... +- compress/... +- text/... +- nex/http/ResponseWriter +- // and many more + +* io.Pipe + +.play -edit pipe/main.go /^func main/,/^}/ + +- synchronous +- in-memory +- no internal buffering + +* iotest + +Package iotest implements Readers and Writers useful mainly for testing. + + // DataErrReader creates a reader that returns (n > 0, EOF) at the end. + func DataErrReader(r io.Reader) io.Reader + + // HalfReader returns a Reader that implements Read + // by reading half as many requested bytes from r. + func HalfReader(r io.Reader) io.Reader + + // OneByteReader returns a Reader that + // implements each non-empty Read by reading one byte from r. + func OneByteReader(r io.Reader) io.Reader + + // TimeoutReader returns ErrTimeout on the second read with no data. + // Subsequent calls to read succeed. + func TimeoutReader(r io.Reader) io.Reader + + // TruncateWriter returns a Writer that writes to w but stops silently after n bytes. + func TruncateWriter(w io.Writer, n int64) io.Writer + +* bufio + +Package bufio implements buffered I/O. It wraps an io.Reader or io.Writer objects. + + type Reader + func NewReader(rd io.Reader) *Reader + func (b *Reader) Discard(n int) (discarded int, err error) + func (b *Reader) Peek(n int) ([]byte, error) + func (b *Reader) Read(p []byte) (n int, err error) + func (b *Reader) ReadByte() (byte, error) + func (b *Reader) ReadBytes(delim byte) ([]byte, error) + func (b *Reader) ReadLine() (line []byte, isPrefix bool, err error) + func (b *Reader) ReadRune() (r rune, size int, err error) + func (b *Reader) ReadSlice(delim byte) (line []byte, err error) + func (b *Reader) ReadString(delim byte) (string, error) + + type Writer + func NewWriter(w io.Writer) *Writer + func (b *Writer) Flush() error + func (b *Writer) ReadFrom(r io.Reader) (n int64, err error) + func (b *Writer) Write(p []byte) (nn int, err error) + func (b *Writer) WriteByte(c byte) error + func (b *Writer) WriteRune(r rune) (size int, err error) + func (b *Writer) WriteString(s string) (int, error) + +* bufio.Scanner + +Utility type to efficiently read independent lines of text from an io.Reader. + +.code -edit scanner/main.go + +* bytes.Buffer + +A handy wrapper around byte slice implementing `io.Reader` and `io.Writer`. + +Useful when you want to use code that takes an io.Writer, and store the results in memory for the later use. + +.play -edit bytesbuffer/main.go + +* *os.File + +- portable +- implements io.Reader, and io.Writer which stream bytes to or from a file on disk +- useful if you don't want to read the whole file into memory + +*ioutil.ReadFile* reads an entire file into memory (as a []byte) in a single call + +- allocates a byte slice of the correct size (no need to Read + append in a loop) +- closes the file +- returns the first error that prevented it from working diff --git a/lectures/09-io/limitreader/main.go b/lectures/09-io/limitreader/main.go new file mode 100644 index 0000000..4903cdc --- /dev/null +++ b/lectures/09-io/limitreader/main.go @@ -0,0 +1,17 @@ +package main + +import ( + "io" + "log" + "os" + "strings" +) + +func main() { + r := strings.NewReader("some io.Reader stream to be read\n") + lr := io.LimitReader(r, 4) + + if _, err := io.Copy(os.Stdout, lr); err != nil { + log.Fatal(err) + } +} diff --git a/lectures/09-io/multireader/main.go b/lectures/09-io/multireader/main.go new file mode 100644 index 0000000..8281004 --- /dev/null +++ b/lectures/09-io/multireader/main.go @@ -0,0 +1,19 @@ +package main + +import ( + "io" + "log" + "os" + "strings" +) + +func main() { + r1 := strings.NewReader("first reader ") + r2 := strings.NewReader("second reader ") + r3 := strings.NewReader("third reader\n") + r := io.MultiReader(r1, r2, r3) + + if _, err := io.Copy(os.Stdout, r); err != nil { + log.Fatal(err) + } +} diff --git a/lectures/09-io/pipe/main.go b/lectures/09-io/pipe/main.go new file mode 100644 index 0000000..0e91b8c --- /dev/null +++ b/lectures/09-io/pipe/main.go @@ -0,0 +1,20 @@ +package main + +import ( + "bytes" + "fmt" + "io" +) + +func main() { + r, w := io.Pipe() + + go func() { + _, _ = fmt.Fprint(w, "some text to be read\n") + _ = w.Close() + }() + + buf := new(bytes.Buffer) + _, _ = buf.ReadFrom(r) + fmt.Print(buf.String()) +} diff --git a/lectures/09-io/readall/main.go b/lectures/09-io/readall/main.go new file mode 100644 index 0000000..6e10263 --- /dev/null +++ b/lectures/09-io/readall/main.go @@ -0,0 +1,20 @@ +package main + +import ( + "fmt" + "io/ioutil" + "log" + "strings" +) + +func main() { + r := strings.NewReader("Go is a general-purpose language " + + "designed with systems programming in mind.") + + b, err := ioutil.ReadAll(r) + if err != nil { + log.Fatal(err) + } + + fmt.Printf("%s", b) +} diff --git a/lectures/09-io/scanner/main.go b/lectures/09-io/scanner/main.go new file mode 100644 index 0000000..eda4979 --- /dev/null +++ b/lectures/09-io/scanner/main.go @@ -0,0 +1,17 @@ +package main + +import ( + "bufio" + "fmt" + "os" +) + +func main() { + scanner := bufio.NewScanner(os.Stdin) + for scanner.Scan() { + fmt.Println(scanner.Text()) // Println will add back the final '\n' + } + if err := scanner.Err(); err != nil { + _, _ = fmt.Fprintln(os.Stderr, "reading standard input:", err) + } +} diff --git a/lectures/09-io/sectionreader/main.go b/lectures/09-io/sectionreader/main.go new file mode 100644 index 0000000..f615ca2 --- /dev/null +++ b/lectures/09-io/sectionreader/main.go @@ -0,0 +1,17 @@ +package main + +import ( + "io" + "log" + "os" + "strings" +) + +func main() { + r := strings.NewReader("some io.Reader stream to be read\n") + s := io.NewSectionReader(r, 5, 17) + + if _, err := io.Copy(os.Stdout, s); err != nil { + log.Fatal(err) + } +} diff --git a/lectures/09-io/sendfile/main.go b/lectures/09-io/sendfile/main.go new file mode 100644 index 0000000..f2608ec --- /dev/null +++ b/lectures/09-io/sendfile/main.go @@ -0,0 +1,58 @@ +package main + +import ( + "io" + "io/ioutil" + "log" + "net/http" + "os" + "strconv" +) + +func main() { + mux := http.NewServeMux() + mux.HandleFunc("/readFile", readFileHandler) + mux.HandleFunc("/copy", copyHandler) + log.Fatal(http.ListenAndServe(":8080", mux)) +} + +func readFileHandler(w http.ResponseWriter, r *http.Request) { + filename := r.URL.Query().Get("file") + data, _ := ioutil.ReadFile(filename) + + // Infer the Content-Type of the file. + contentType := http.DetectContentType(data[:512]) + + // Get the file size. + fileSize := strconv.FormatInt(int64(len(data)), 10) + + // Send the headers. + w.Header().Set("Content-Disposition", "attachment; filename="+filename) + w.Header().Set("Content-Type", contentType) + w.Header().Set("Content-Length", fileSize) + + _, _ = w.Write(data) +} + +func copyHandler(w http.ResponseWriter, r *http.Request) { + filename := r.URL.Query().Get("file") + f, _ := os.Open(filename) + defer func() { _ = f.Close() }() + + // Infer the Content-Type of the file. + filePrefix := make([]byte, 512) + _, _ = f.Read(filePrefix) + contentType := http.DetectContentType(filePrefix) + + // Get the file size. + fstat, _ := f.Stat() + fileSize := strconv.FormatInt(fstat.Size(), 10) + + // Send the headers. + w.Header().Set("Content-Disposition", "attachment; filename="+filename) + w.Header().Set("Content-Type", contentType) + w.Header().Set("Content-Length", fileSize) + + _, _ = f.Seek(0, 0) + _, _ = io.Copy(w, f) +} diff --git a/lectures/09-io/teereader/main.go b/lectures/09-io/teereader/main.go new file mode 100644 index 0000000..cd9efdc --- /dev/null +++ b/lectures/09-io/teereader/main.go @@ -0,0 +1,29 @@ +package main + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "log" + "strings" +) + +func main() { + r := strings.NewReader("some io.Reader stream to be read\n") + var buf bytes.Buffer + tee := io.TeeReader(r, &buf) + + printall := func(r io.Reader) { + b, err := ioutil.ReadAll(r) + if err != nil { + log.Fatal(err) + } + + fmt.Printf("%s", b) + } + + printall(tee) + printall(&buf) + +}