Added input, added C89 support, fixed numerous issues

Dockerfile

@@ -1,14 +1,18 @@
 FROM jupyter/minimal-notebook
-MAINTAINER Brendan Rius <[email protected]>
+MAINTAINER Xaver Klemenschits <[email protected]>
 
 USER root
 
+# Install vim and ssh
+RUN apt-get update
+RUN apt-get install -y vim openssh-client
+
 WORKDIR /tmp
 
 COPY ./ jupyter_c_kernel/
 
-RUN pip install --no-cache-dir jupyter_c_kernel/
-RUN cd jupyter_c_kernel && install_c_kernel --user
+RUN pip install --no-cache-dir -e jupyter_c_kernel/ > piplog.txt
+RUN cd jupyter_c_kernel && install_c_kernel --user > installlog.txt
 
 WORKDIR /home/$NB_USER/
 

MANIFEST.in

@@ -1,2 +1,3 @@
 include jupyter_c_kernel/resources/master.c
+include jupyter_c_kernel/resources/stdio_wrap.h
 include README.md LICENSE.txt

README.md

@@ -1,12 +1,25 @@
-# Minimal C kernel for Jupyter
+# C kernel for Jupyter
+
+This project was forked from [https://github.com/brendan-rius/jupyter-c-kernel](brendan-rius/jupyter-c-kernel) as that project seems to have been abandoned. (PR is pending)
+
+This project includes fixes to many issues reported in [https://github.com/brendan-rius/jupyter-c-kernel](brendan-rius/jupyter-c-kernel), as well as the following additional features:
+
+* Option for buffered output to mimic command line behaviour (useful for teaching, default is on)
+* Command line input via `scanf` and `getchar`
+* Support for `C89`/`ANSI C` (all newer versions were already supported and still are)
+
+Following limitations compared to command line execution exist:
+
+* Input is always buffered due to limitations of the jupyter interface
+* When using `-ansi` or `-std=C89`, glibc still has to support at least `C99` for the interfacing with jupyter (this should not be an issue on an OS made after 2000)
 
 ## Use with Docker (recommended)
 
- * `docker pull brendanrius/jupyter-c-kernel`
- * `docker run -p 8888:8888 brendanrius/jupyter-c-kernel`
- * Copy the given URL containing the token, and browse to it. For instance:
- 
- ```
+* `docker pull xaverklemenschits/jupyter-c-kernel`
+* `docker run -p 8888:8888 xaverklemenschits/jupyter-c-kernel`
+* Copy the given URL containing the token, and browse to it. For instance:
+
+ ```bash
  Copy/paste this URL into your browser when you connect for the first time,
  to login with a token:
     http://localhost:8888/?token=66750c80bd0788f6ba15760aadz53beb9a9fb4cf8ac15ce8
@@ -16,17 +29,22 @@
 
 Works only on Linux and OS X. Windows is not supported yet. If you want to use this project on Windows, please use Docker.
 
-
- * Make sure you have the following requirements installed:
+* Make sure you have the following requirements installed:
   * gcc
   * jupyter
   * python 3
   * pip
 
-### Step-by-step:
- * `pip install jupyter-c-kernel`
- * `install_c_kernel`
- * `jupyter-notebook`. Enjoy!
+### Step-by-step
+
+```bash
+git clone https://github.com/XaverKlemenschits/jupyter-c-kernel.git
+cd jupyter-c-kernel
+pip install -e .  # for system install: sudo install .
+cd jupyter_c_kernel && install_c_kernel --user # for sys install: sudo install_c_kernel
+# now you can start the notebook
+jupyter notebook
+```
 
 ## Example of notebook
 
@@ -47,9 +65,10 @@ change the code in real-time in Docker. For that, just run the docker box like
 that:
 
 ```bash
-git clone https://github.com/brendan-rius/jupyter-c-kernel.git
+git clone https://github.com/XaverKlemenschits/jupyter-c-kernel.git
 cd jupyter-c-kernel
-docker run -v $(pwd):/jupyter/jupyter_c_kernel/ -p 8888:8888 brendanrius/jupyter-c-kernel
+docker build -t myName/jupyter .
+docker run -v $(pwd):/tmp/jupyter_c_kernel/ -p 8888:8888 myName/jupyter
 ```
 
 This clones the source, run the kernel, and binds the current folder (the one

example-notebook.ipynb

@@ -10,9 +10,7 @@
   {
    "cell_type": "code",
    "execution_count": 1,
-   "metadata": {
-    "collapsed": false
-   },
+   "metadata": {},
    "outputs": [
     {
      "name": "stderr",
@@ -32,6 +30,7 @@
     "\n",
     "int main() {\n",
     "    printf(\"Hello world\\n\");\n",
+    "    return 0;\n",
     "}"
    ]
   },
@@ -52,9 +51,7 @@
   {
    "cell_type": "code",
    "execution_count": 2,
-   "metadata": {
-    "collapsed": false
-   },
+   "metadata": {},
    "outputs": [
     {
      "name": "stderr",
@@ -98,9 +95,7 @@
   {
    "cell_type": "code",
    "execution_count": 3,
-   "metadata": {
-    "collapsed": false
-   },
+   "metadata": {},
    "outputs": [
     {
      "name": "stderr",
@@ -142,14 +137,14 @@
   "kernelspec": {
    "display_name": "C",
    "language": "c",
-   "name": "c_kernel"
+   "name": "c"
   },
   "language_info": {
-   "file_extension": "c",
+   "file_extension": ".c",
    "mimetype": "text/plain",
-   "name": "c"
+   "name": "text/x-c++src"
   }
  },
  "nbformat": 4,
- "nbformat_minor": 0
+ "nbformat_minor": 1
 }

jupyter_c_kernel/install_c_kernel

@@ -29,7 +29,7 @@ def install_my_kernel_spec(user=True, prefix=None):
         # TODO: Copy resources once they're specified
 
         print('Installing IPython kernel spec')
-        KernelSpecManager().install_kernel_spec(td, 'c', user=user, replace=True, prefix=prefix)
+        KernelSpecManager().install_kernel_spec(td, 'c', user=user, prefix=prefix)
 
 
 def _is_root():

jupyter_c_kernel/kernel.py

@@ -14,16 +14,19 @@ class RealTimeSubprocess(subprocess.Popen):
     A subprocess that allows to read its stdout and stderr in real time
     """
 
-    def __init__(self, cmd, write_to_stdout, write_to_stderr):
+    inputRequest = "<inputRequest>"
+
+    def __init__(self, cmd, write_to_stdout, write_to_stderr, read_from_stdin):
         """
         :param cmd: the command to execute
         :param write_to_stdout: a callable that will be called with chunks of data from stdout
         :param write_to_stderr: a callable that will be called with chunks of data from stderr
         """
         self._write_to_stdout = write_to_stdout
         self._write_to_stderr = write_to_stderr
+        self._read_from_stdin = read_from_stdin
 
-        super().__init__(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=0)
+        super().__init__(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, bufsize=0)
 
         self._stdout_queue = Queue()
         self._stdout_thread = Thread(target=RealTimeSubprocess._enqueue_output, args=(self.stdout, self._stdout_queue))
@@ -58,12 +61,28 @@ def read_all_from_queue(queue):
                 size -= 1
             return res
 
-        stdout_contents = read_all_from_queue(self._stdout_queue)
-        if stdout_contents:
-            self._write_to_stdout(stdout_contents)
         stderr_contents = read_all_from_queue(self._stderr_queue)
         if stderr_contents:
-            self._write_to_stderr(stderr_contents)
+            self._write_to_stderr(stderr_contents.decode())
+
+        stdout_contents = read_all_from_queue(self._stdout_queue)
+        if stdout_contents:
+            contents = stdout_contents.decode()
+            # if there is input request, make output and then
+            # ask frontend for input
+            start = contents.find(self.__class__.inputRequest)
+            if(start >= 0):
+                contents = contents.replace(self.__class__.inputRequest, '')
+                if(len(contents) > 0):
+                    self._write_to_stdout(contents)
+                readLine = ""
+                while(len(readLine) == 0):
+                    readLine = self._read_from_stdin()
+                # need to add newline since it is not captured by frontend
+                readLine += "\n"
+                self.stdin.write(readLine.encode())
+            else:
+                self._write_to_stdout(contents)
 
 
 class CKernel(Kernel):
@@ -72,24 +91,41 @@ class CKernel(Kernel):
     language = 'c'
     language_version = 'C11'
     language_info = {'name': 'c',
-                     'mimetype': 'text/plain',
+                     'mimetype': 'text/x-csrc',
                      'file_extension': '.c'}
     banner = "C kernel.\n" \
              "Uses gcc, compiles in C11, and creates source code files and executables in temporary folder.\n"
 
+    main_head = "#include <stdio.h>\n" \
+            "#include <math.h>\n" \
+            "int main(){\n"
+
+    main_foot = "\nreturn 0;\n}"
+
     def __init__(self, *args, **kwargs):
         super(CKernel, self).__init__(*args, **kwargs)
+        self._allow_stdin = True
+        self.readOnlyFileSystem = False
+        self.bufferedOutput = True
+        self.linkMaths = True # always link math library
+        self.wAll = True # show all warnings by default
+        self.wError = False # but keep comipiling for warnings
+        self.standard = "c11" # default standard if none is specified
         self.files = []
         mastertemp = tempfile.mkstemp(suffix='.out')
         os.close(mastertemp[0])
         self.master_path = mastertemp[1]
-        filepath = path.join(path.dirname(path.realpath(__file__)), 'resources', 'master.c')
+        self.resDir = path.join(path.dirname(path.realpath(__file__)), 'resources')
+        filepath = path.join(self.resDir, 'master.c')
         subprocess.call(['gcc', filepath, '-std=c11', '-rdynamic', '-ldl', '-o', self.master_path])
 
     def cleanup_files(self):
         """Remove all the temporary files created by the kernel"""
+        # keep the list of files create in case there is an exception
+        # before they can be deleted as usual
         for file in self.files:
-            os.remove(file)
+            if(os.path.exists(file)):
+                os.remove(file)
         os.remove(self.master_path)
 
     def new_temp_file(self, **kwargs):
@@ -107,13 +143,27 @@ def _write_to_stdout(self, contents):
     def _write_to_stderr(self, contents):
         self.send_response(self.iopub_socket, 'stream', {'name': 'stderr', 'text': contents})
 
+    def _read_from_stdin(self):
+        return self.raw_input()
+
     def create_jupyter_subprocess(self, cmd):
         return RealTimeSubprocess(cmd,
-                                  lambda contents: self._write_to_stdout(contents.decode()),
-                                  lambda contents: self._write_to_stderr(contents.decode()))
+                                  self._write_to_stdout,
+                                  self._write_to_stderr,
+                                  self._read_from_stdin)
 
     def compile_with_gcc(self, source_filename, binary_filename, cflags=None, ldflags=None):
-        cflags = ['-std=c11', '-fPIC', '-shared', '-rdynamic'] + cflags
+        cflags = ['-pedantic', '-fPIC', '-shared', '-rdynamic'] + cflags
+        if self.linkMaths:
+            cflags = cflags + ['-lm']
+        if self.wError:
+            cflags = cflags + ['-Werror']
+        if self.wAll:
+            cflags = cflags + ['-Wall']
+        if self.readOnlyFileSystem:
+            cflags = ['-DREAD_ONLY_FILE_SYSTEM'] + cflags
+        if self.bufferedOutput:
+            cflags = ['-DBUFFERED_OUTPUT'] + cflags
         args = ['gcc', source_filename] + cflags + ['-o', binary_filename] + ldflags
         return self.create_jupyter_subprocess(args)
 
@@ -123,9 +173,16 @@ def _filter_magics(self, code):
                   'ldflags': [],
                   'args': []}
 
+        actualCode = ''
+
         for line in code.splitlines():
             if line.startswith('//%'):
-                key, value = line[3:].split(":", 2)
+                magicSplit = line[3:].split(":", 2)
+                if(len(magicSplit) < 2):
+                    self._write_to_stderr("[C kernel] Magic line starting with '//%' is missing a semicolon, ignoring.")
+                    continue
+
+                key, value = magicSplit
                 key = key.strip().lower()
 
                 if key in ['ldflags', 'cflags']:
@@ -136,12 +193,45 @@ def _filter_magics(self, code):
                     for argument in re.findall(r'(?:[^\s,"]|"(?:\\.|[^"])*")+', value):
                         magics['args'] += [argument.strip('"')]
 
-        return magics
+                # always add empty line, so line numbers don't change
+                actualCode += '\n'
+
+            # keep lines which did not contain magics
+            else:
+                actualCode += line + '\n'
+
+        # add default standard if cflags does not contain one
+        if not any(item.startswith('-std=') for item in magics["cflags"]):
+            magics["cflags"] += ["-std=" + self.standard]
+
+        return magics, actualCode
+
+    # check whether int main() is specified, if not add it around the code
+    # also add common magics like -lm
+    def _add_main(self, magics, code):
+        # remove comments
+        tmpCode = re.sub(r"//.*", "", code)
+        tmpCode = re.sub(r"/\*.*?\*/", "", tmpCode, flags=re.M|re.S)
+
+        x = re.search(r"int\s+main\s*\(", tmpCode)
+
+        if not x:
+            code = self.main_head + code + self.main_foot
+            magics['cflags'] += ['-lm']
+
+        return magics, code
 
     def do_execute(self, code, silent, store_history=True,
-                   user_expressions=None, allow_stdin=False):
+                   user_expressions=None, allow_stdin=True):
+
+        magics, code = self._filter_magics(code)
 
-        magics = self._filter_magics(code)
+        magics, code = self._add_main(magics, code)
+
+        # replace stdio with wrapped version
+        headerDir = "\"" + self.resDir + "/stdio_wrap.h" + "\""
+        code = code.replace("<stdio.h>", headerDir)
+        code = code.replace("\"stdio.h\"", headerDir)
 
         with self.new_temp_file(suffix='.c') as source_file:
             source_file.write(code)
@@ -155,14 +245,28 @@ def do_execute(self, code, silent, store_history=True,
                     self._write_to_stderr(
                             "[C kernel] GCC exited with code {}, the executable will not be executed".format(
                                     p.returncode))
+
+                    # delete source files before exit
+                    os.remove(source_file.name)
+                    os.remove(binary_file.name)
+
                     return {'status': 'ok', 'execution_count': self.execution_count, 'payload': [],
                             'user_expressions': {}}
 
         p = self.create_jupyter_subprocess([self.master_path, binary_file.name] + magics['args'])
         while p.poll() is None:
             p.write_contents()
+
+        # wait for threads to finish, so output is always shown
+        p._stdout_thread.join()
+        p._stderr_thread.join()
+
         p.write_contents()
 
+        # now remove the files we have just created
+        os.remove(source_file.name)
+        os.remove(binary_file.name)
+
         if p.returncode != 0:
             self._write_to_stderr("[C kernel] Executable exited with code {}".format(p.returncode))
         return {'status': 'ok', 'execution_count': self.execution_count, 'payload': [], 'user_expressions': {}}