From 666f355c6939a40d1ab035f237e0c84ec790a6ef Mon Sep 17 00:00:00 2001 From: Fedor Korotkiy Date: Thu, 23 Apr 2020 14:27:50 +0300 Subject: [PATCH] Low-level lecture --- lectures/09-lowlevel/bzip/bzip2.c | 32 +++++++ lectures/09-lowlevel/bzip/bzip2.go | 111 ++++++++++++++++++++++++ lectures/09-lowlevel/bzip/bzip2_test.go | 40 +++++++++ lectures/09-lowlevel/lecture.slide | 84 ++++++++++++++++++ lectures/09-lowlevel/unsafeptr/main.go | 36 ++++++++ 5 files changed, 303 insertions(+) create mode 100644 lectures/09-lowlevel/bzip/bzip2.c create mode 100644 lectures/09-lowlevel/bzip/bzip2.go create mode 100644 lectures/09-lowlevel/bzip/bzip2_test.go create mode 100644 lectures/09-lowlevel/lecture.slide create mode 100644 lectures/09-lowlevel/unsafeptr/main.go diff --git a/lectures/09-lowlevel/bzip/bzip2.c b/lectures/09-lowlevel/bzip/bzip2.c new file mode 100644 index 0000000..985869e --- /dev/null +++ b/lectures/09-lowlevel/bzip/bzip2.c @@ -0,0 +1,32 @@ +// Copyright © 2016 Alan A. A. Donovan & Brian W. Kernighan. +// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ + +// See page 362. +// +// The version of this program that appeared in the first and second +// printings did not comply with the proposed rules for passing +// pointers between Go and C, described here: +// https://github.com/golang/proposal/blob/master/design/12416-cgo-pointers.md +// +// The version below, which appears in the third printing, +// has been corrected. See bzip2.go for explanation. + +//!+ +/* This file is gopl.io/ch13/bzip/bzip2.c, */ +/* a simple wrapper for libbzip2 suitable for cgo. */ +#include + +int bz2compress(bz_stream *s, int action, + char *in, unsigned *inlen, char *out, unsigned *outlen) { + s->next_in = in; + s->avail_in = *inlen; + s->next_out = out; + s->avail_out = *outlen; + int r = BZ2_bzCompress(s, action); + *inlen -= s->avail_in; + *outlen -= s->avail_out; + s->next_in = s->next_out = NULL; + return r; +} + +//!- diff --git a/lectures/09-lowlevel/bzip/bzip2.go b/lectures/09-lowlevel/bzip/bzip2.go new file mode 100644 index 0000000..278b238 --- /dev/null +++ b/lectures/09-lowlevel/bzip/bzip2.go @@ -0,0 +1,111 @@ +// Copyright © 2016 Alan A. A. Donovan & Brian W. Kernighan. +// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ + +// See page 362. +// +// The version of this program that appeared in the first and second +// printings did not comply with the proposed rules for passing +// pointers between Go and C, described here: +// https://github.com/golang/proposal/blob/master/design/12416-cgo-pointers.md +// +// The rules forbid a C function like bz2compress from storing 'in' +// and 'out' (pointers to variables allocated by Go) into the Go +// variable 's', even temporarily. +// +// The version below, which appears in the third printing, has been +// corrected. To comply with the rules, the bz_stream variable must +// be allocated by C code. We have introduced two C functions, +// bz2alloc and bz2free, to allocate and free instances of the +// bz_stream type. Also, we have changed bz2compress so that before +// it returns, it clears the fields of the bz_stream that contain +// pointers to Go variables. + +//!+ + +// Package bzip provides a writer that uses bzip2 compression (bzip.org). +package bzip + +/* +#cgo CFLAGS: -I/usr/include +#cgo LDFLAGS: -L/usr/lib -lbz2 +#include +#include +bz_stream* bz2alloc() { return calloc(1, sizeof(bz_stream)); } +int bz2compress(bz_stream *s, int action, + char *in, unsigned *inlen, char *out, unsigned *outlen); +void bz2free(bz_stream* s) { free(s); } +*/ +import "C" + +import ( + "io" + "unsafe" +) + +type writer struct { + w io.Writer // underlying output stream + stream *C.bz_stream + outbuf [64 * 1024]byte +} + +// NewWriter returns a writer for bzip2-compressed streams. +func NewWriter(out io.Writer) io.WriteCloser { + const blockSize = 9 + const verbosity = 0 + const workFactor = 30 + w := &writer{w: out, stream: C.bz2alloc()} + C.BZ2_bzCompressInit(w.stream, blockSize, verbosity, workFactor) + return w +} + +//!- + +//!+write +func (w *writer) Write(data []byte) (int, error) { + if w.stream == nil { + panic("closed") + } + var total int // uncompressed bytes written + + for len(data) > 0 { + inlen, outlen := C.uint(len(data)), C.uint(cap(w.outbuf)) + C.bz2compress(w.stream, C.BZ_RUN, + (*C.char)(unsafe.Pointer(&data[0])), &inlen, + (*C.char)(unsafe.Pointer(&w.outbuf)), &outlen) + total += int(inlen) + data = data[inlen:] + if _, err := w.w.Write(w.outbuf[:outlen]); err != nil { + return total, err + } + } + return total, nil +} + +//!-write + +//!+close +// Close flushes the compressed data and closes the stream. +// It does not close the underlying io.Writer. +func (w *writer) Close() error { + if w.stream == nil { + panic("closed") + } + defer func() { + C.BZ2_bzCompressEnd(w.stream) + C.bz2free(w.stream) + w.stream = nil + }() + for { + inlen, outlen := C.uint(0), C.uint(cap(w.outbuf)) + r := C.bz2compress(w.stream, C.BZ_FINISH, nil, &inlen, + (*C.char)(unsafe.Pointer(&w.outbuf)), &outlen) + if _, err := w.w.Write(w.outbuf[:outlen]); err != nil { + return err + } + if r == C.BZ_STREAM_END { + return nil + } + } +} + +//!-close diff --git a/lectures/09-lowlevel/bzip/bzip2_test.go b/lectures/09-lowlevel/bzip/bzip2_test.go new file mode 100644 index 0000000..c52aba0 --- /dev/null +++ b/lectures/09-lowlevel/bzip/bzip2_test.go @@ -0,0 +1,40 @@ +// Copyright © 2016 Alan A. A. Donovan & Brian W. Kernighan. +// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ + +package bzip_test + +import ( + "bytes" + "compress/bzip2" // reader + "io" + "testing" + + "gopl.io/ch13/bzip" // writer +) + +func TestBzip2(t *testing.T) { + var compressed, uncompressed bytes.Buffer + w := bzip.NewWriter(&compressed) + + // Write a repetitive message in a million pieces, + // compressing one copy but not the other. + tee := io.MultiWriter(w, &uncompressed) + for i := 0; i < 1000000; i++ { + io.WriteString(tee, "hello") + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + + // Check the size of the compressed stream. + if got, want := compressed.Len(), 255; got != want { + t.Errorf("1 million hellos compressed to %d bytes, want %d", got, want) + } + + // Decompress and compare with original. + var decompressed bytes.Buffer + io.Copy(&decompressed, bzip2.NewReader(&compressed)) + if !bytes.Equal(uncompressed.Bytes(), decompressed.Bytes()) { + t.Error("decompression yielded a different message") + } +} diff --git a/lectures/09-lowlevel/lecture.slide b/lectures/09-lowlevel/lecture.slide new file mode 100644 index 0000000..73fffd5 --- /dev/null +++ b/lectures/09-lowlevel/lecture.slide @@ -0,0 +1,84 @@ +low-level +Лекция 8 + +Короткий Фёдор + +* Low Level Programming + +* package unsafe + + import "unsafe" + + fmt.Println(unsafe.Sizeof(float64(0))) // "8" + +- `unsafe` выглядит как обычный пакет +- Но реализован в компиляторе + +* package unsafe + + type Pointer *ArbitraryType + +- Два преобразования + + *T -> unsafe.Pointer + unsafe.Pointer -> *T + +- `unsafe.Pointer` - это _настоящий_ `void*` + + func Float64bits(f float64) uint64 { + return *(*uint64)(unsafe.Pointer(&f)) + } + +* unsafe.Pointer + +.code unsafeptr/main.go /var/,/42/ + +Действия с указателями дожны выполняться *атомарно*. + +.code unsafeptr/main.go /\+wrong/,/\-wrong/ + +Иначе сборщик мусора освободит память используемого объекта. + + pT := uintptr(unsafe.Pointer(new(T))) // NOTE: wrong! + +* unsafe tricks + + // A Builder is used to efficiently build a string using Write methods. + // It minimizes memory copying. The zero value is ready to use. + // Do not copy a non-zero Builder. + type Builder struct { + buf []byte + } + + // String returns the accumulated string. + func (b *Builder) String() string { + return *(*string)(unsafe.Pointer(&b.buf)) + } + + // *-----*-----*-----* + // * ptr * len * cap * // []byte + // *-----*-----*-----* + // + // *-----*-----* + // * ptr * len * // string + // *-----*-----* + +* cgo + +.code bzip/bzip2.c /include/,/^}/ + +* cgo + +.code bzip/bzip2.go /package/,/import/ + +.code bzip/bzip2.go /type writer/,/^}/ + +* cgo + +.code bzip/bzip2.go /type writer/,/^}/ + +.code bzip/bzip2.go /func NewWriter/,/^}/ + +* cgo + +.code bzip/bzip2.go /func .* Write/,/^}/ diff --git a/lectures/09-lowlevel/unsafeptr/main.go b/lectures/09-lowlevel/unsafeptr/main.go new file mode 100644 index 0000000..fb7a55e --- /dev/null +++ b/lectures/09-lowlevel/unsafeptr/main.go @@ -0,0 +1,36 @@ +// Copyright © 2016 Alan A. A. Donovan & Brian W. Kernighan. +// License: https://creativecommons.org/licenses/by-nc-sa/4.0/ + +// See page 357. + +// Package unsafeptr demonstrates basic use of unsafe.Pointer. +package main + +import ( + "fmt" + "unsafe" +) + +func main() { + var x struct { + a bool + b int16 + c []int + } + + // equivalent to pb := &x.b + pb := (*int16)(unsafe.Pointer( + uintptr(unsafe.Pointer(&x)) + unsafe.Offsetof(x.b))) + *pb = 42 + + fmt.Println(x.b) // "42" +} + +/* +//!+wrong OMIT +// NOTE: subtly incorrect! +tmp := uintptr(unsafe.Pointer(&x)) + unsafe.Offsetof(x.b) +pb := (*int16)(unsafe.Pointer(tmp)) +*pb = 42 +//!-wrong OMIT +*/