Docker: Stop collecting usage stats for streamlit;

Add Ace editor for preview;
Add parcing header of .snp

Dockerfile

@@ -5,6 +5,6 @@ RUN pip install -r requirements.txt
 EXPOSE 8501
 COPY . /app
 ENTRYPOINT ["streamlit", "run"]
-CMD ["./source/main.py", "--browser.serverAddress=0.0.0.0"]
+CMD ["./source/main.py", "--browser.serverAddress=0.0.0.0", "--browser.gatherUsageStats=false"]
 # docker build -t calc-q-factor:latest .
 # docker run -p 8501:8501 calc-q-factor:latest

TODO.md

@@ -5,10 +5,10 @@
     * What is Q circle?
     * What data formats are supported?
 2. [x] Add validation of separator + convertor to std backend input
-3. Should we apply corrections for coupling losses? - yes, please add this option.
+3. [x] Should we apply corrections for coupling losses? - yes, please add this option.
 4. [x] Draw continuous Q circle on a Smith chart using coefficients a[0..2]
 5. [x] Add axes labels to a Smith chart
-6. Pretty-print results and errors (7 digits after dot). Try latex output.
+6. [x] Pretty-print results and errors (7 digits after dot). Try latex output.
 7. Advanced output options (only frontend):
     * Option to choose output values precision
 8. Add approximation for second chart (abs(S11) from f)
@@ -32,4 +32,4 @@
 13. Do we need to calculate systematic errors? - yes, if its not too hard.
 14. Add direct support for output files from different vna models
 15. Make charts more interactive
-16. Make an option to pass the whole program to .html site as a iframe
+16. Make an option to pass the whole program to .html site as a iframe? - no, Just docker container?

requirements.txt

@@ -3,4 +3,5 @@ streamlit
 matplotlib
 numpy
 # sympy?
-streamlit_echarts
+streamlit_echarts
+streamlit-ace

source/frontend/front.py

@@ -1,3 +1,5 @@
+from streamlit_ace import st_ace
+from streamlit_echarts import st_echarts
 import math
 import streamlit as st
 import matplotlib.pyplot as plt
@@ -6,14 +8,6 @@ import numpy as np
 XLIM = [-1.1, 1.1]
 YLIM = [-1.1, 1.1]
 
-
-def round_up(x, n=7):
-    if x == 0:
-        return 0
-    deg = math.floor(math.log(abs(x), 10))
-    return (10 ** deg) * round(x / (10 ** deg), n - 1)
-
-
 def circle(ax, x, y, radius, color='#1946BA'):
     from matplotlib.patches import Ellipse
     drawn_circle = Ellipse((x, y), radius * 2, radius * 2, clip_on=False,
@@ -50,15 +44,19 @@ def plot_data(r, i, g):
     #
     ax.set_xlim(XLIM)
     ax.set_ylim(YLIM)
-    st.pyplot(fig)
+    try:
+        st.pyplot(fig)
+    except:
+        st.write("Plot size is too big, check your input")
+
+
+interval_range = (0, 100)
+interval_start, interval_end = 0, 0
 
 
-from streamlit_echarts import st_echarts, JsCode
-interval_range = (0,100)
-interval_start,interval_end=0,0
-def plot_interact_abs_from_f(f,r,i):
+def plot_interact_abs_from_f(f, r, i):
     abs_S = list((r[n] ** 2 + i[n] ** 2)**0.5 for n in range(len(r)))
-    global interval_range,interval_start,interval_end
+    global interval_range, interval_start, interval_end
     # echarts for datazoom https://discuss.streamlit.io/t/streamlit-echarts/3655
     # datazoom https://echarts.apache.org/examples/en/editor.html?c=line-draggable&lang=ts
     # axis pointer values https://echarts.apache.org/en/option.html#axisPointer
@@ -66,29 +64,35 @@ def plot_interact_abs_from_f(f,r,i):
         "xAxis": {
             "type": "category",
             "data": f,
+            "name": "Hz",
+            "nameTextStyle": {"fontSize": 16},
+            "axisLabel": {"fontSize": 16}
         },
         "yAxis": {
             "type": "value",
-            "name":"abs(S)",
+            "name": "abs(S)",
+            "nameTextStyle": {"fontSize": 16},
+            "axisLabel": {"fontSize": 16}
         },
-        "series": [{"data": abs_S, "type": "line", "name":"abs(S)"}],
-        "dataZoom": [{"type": "slider", "start": 0, "end": 100}],
+        "series": [{"data": abs_S, "type": "line", "name": "abs(S)"}],
+        "height": 300,
+        "dataZoom": [{"type": "slider", "start": 0, "end": 100, "height": 100, "bottom": 10}],
         "tooltip": {
-            "trigger":"axis",
+            "trigger": "axis",
             "axisPointer": {
                 "type": 'cross',
                 # "label": {
-                    # "show":"true",
+                # "show":"true",
                 # "formatter": JsCode(
-            # "function(info){return info.value;};"
-        # ).js_code
+                # "function(info){return info.value;};"
+                # ).js_code
                 # }
             }
         },
-        "toolbox":{
+        "toolbox": {
             "feature": {
-                "dataView": { "show": "true", "readOnly": "true" },
-                "restore": { "show": "true" },
+                # "dataView": { "show": "true", "readOnly": "true" },
+                "restore": {"show": "true"},
             }
         },
     }
@@ -102,43 +106,34 @@ def plot_interact_abs_from_f(f,r,i):
     if interval_range is None:
         interval_range = (0, 100)
 
-    n=len(f)
-    interval_start,interval_end=(int(n*interval_range[id]*0.01) for id in (0,1))
+    n = len(f)
+    interval_start, interval_end = (
+        int(n*interval_range[id]*0.01) for id in (0, 1))
+
 
 def plot_ref_from_f(f, r, i):
     fig = plt.figure(figsize=(10, 10))
     abs_S = list((r[n] ** 2 + i[n] ** 2)**0.5 for n in range(len(r)))
-    xlim = [min(f) - abs(max(f) - min(f)) * 0.1, max(f) + abs(max(f) - min(f)) * 0.1]
-    ylim = [min(abs_S) - abs(max(abs_S) - min(abs_S)) * 0.5, max(abs_S) + abs(max(abs_S) - min(abs_S)) * 0.5]
+    xlim = [min(f) - abs(max(f) - min(f)) * 0.1,
+            max(f) + abs(max(f) - min(f)) * 0.1]
+    ylim = [min(abs_S) - abs(max(abs_S) - min(abs_S)) * 0.5,
+            max(abs_S) + abs(max(abs_S) - min(abs_S)) * 0.5]
     ax = fig.add_subplot()
     ax.set_xlim(xlim)
     ax.set_ylim(ylim)
     ax.grid(which='major', color='k', linewidth=1)
     ax.grid(which='minor', color='grey', linestyle=':', linewidth=0.5)
     plt.xlabel(r'$f,\; 1/c$', color='gray', fontsize=16, fontname="Cambria")
-    plt.ylabel('$|\Gamma|$', color='gray', fontsize=16, fontname="Cambria")
-    plt.title('Modulus of reflection coefficient from frequency', fontsize=24, fontname="Cambria")
-    
+    plt.ylabel('$|S|$', color='gray', fontsize=16, fontname="Cambria")
+    plt.title('Absolute value of reflection coefficient from frequency',
+              fontsize=24, fontname="Cambria")
+
     ax.plot(f, abs_S, '+', ms=10, mew=2, color='#1946BA')
     st.pyplot(fig)
 
 
 def run(calc_function):
-    global interval_range,interval_start,   interval_end
 
-    data = []
-    uploaded_file = st.file_uploader('Upload a csv')
-    if uploaded_file is not None:
-        data = uploaded_file.readlines()
-
-    col1, col2 = st.columns(2)
-
-    select_data_format = col1.selectbox('Choose data format from a list',
-                                        ['Frequency, Re(S11), Im(S11)', 'Frequency, Re(Zin), Im(Zin)'])
-
-    select_separator = col2.selectbox('Choose separator', ['" "', '","', '";"'])
-    select_coupling_losses = st.checkbox('Apply corrections for coupling losses (lossy coupling)')
-    
     def is_float(element) -> bool:
         try:
             float(element)
@@ -149,71 +144,162 @@ def run(calc_function):
         except ValueError:
             return False
 
-    def unpack_data(data):
-        nonlocal select_separator
-        nonlocal select_data_format
-        f, r, i = [], [], []
-        if select_data_format == 'Frequency, Re(S11), Im(S11)':
+    # to utf-8
+    def read_data(data):
+        for x in range(len(data)):
+            if type(data[x]) == bytes:
+                try:
+                    data[x] = data[x].decode('utf-8-sig', 'ignore')
+                except:
+                    return 'Not an utf-8-sig line №: ' + str(x)
+        return 'data read: success'
+
+    # for Touchstone .snp format
+    def parse_heading(data):
+        nonlocal data_format_snp
+        if data_format_snp:
             for x in range(len(data)):
-                # print(select_separator)
-                select_separator = select_separator.replace('\"', '')
-                if type(data[x])==bytes:
-                    # print('f')
-                    try: 
-                        data[x]=data[x].decode('utf-8-sig', 'ignore')
-                    except:
-                        return f, r, i, 'Not an utf-8-sig line №: ' + str(x)
-
-                if select_separator == " ":
-                    tru = data[x].split()
-                else:
-                    data[x] = data[x].replace(select_separator, ' ')
-                    tru = data[x].split()
+                if data[x][0]=='#':
+                    line = data[x].split()
+                    if len(line)== 6:
+                        repr_map = {"RI":0,"MA":1, "DB":2}
+                        para_map = {"S":0,"Z":1}
+                        hz_map = {"GHz":10**9,"MHz":10**6,"KHz":10**3,"Hz":1}
+                        hz,measurement_parameter,data_representation,_r,ref_resistance=line[1:]
+                        try:
+                            return hz_map[hz], para_map[measurement_parameter], repr_map[data_representation], ref_resistance
+                        except:
+                            break
+                    break
+            data_format_snp = False
+        return 1, 0, 0, 50
+
 
-                if len(tru) != 3:
-                    return f, r, i, 'Can\'t parse line №: ' + str(x)
-                a, b, c = (y for y in tru)
+    def unpack_data(data, input_start_line, input_end_line, hz):
+        nonlocal select_measurement_parameter
+        nonlocal select_data_representation
+        f, r, i = [], [], []
+        for x in range(input_start_line-1, input_end_line):
+            if len(data[x])<2 or data[x][0]== '!' or data[x][0]=='#' or data[x][0]=='%' or data[x][0]=='/':
+                # first is a comment line according to .snp documentation,
+                # others detects comments in various languages
+                continue
+            data[x] = data[x].replace(';', ' ').replace(',', ' ')
+            line = data[x].split()
+            # always at least 3 values for single data point
+            if len(line) < 3:
+                return f, r, i, 'Can\'t parse line №: ' + str(x) + ',\n not enough arguments (less than 3)'
+            if select_measurement_parameter == 'S':
+                a, b, c = (y for y in line)
                 if not ((is_float(a)) or (is_float(b)) or (is_float(c))):
-                    return f, r, i, 'Your data isn\'t numerical type. Error on line: ' + str(x)
-                f.append(float(a))  # frequency
-                r.append(float(b))  # Re of S11
-                i.append(float(c))  # Im of S11
-        else:
-            return f, r, i, 'Wrong data format'
+                    return f, r, i, 'Wrong data type, expected number. Error on line: ' + str(x)
+            else:
+                return f, r, i, 'Wrong data format'
+
+            f.append(float(a)*hz)  # frequency
+            r.append(float(b))  # Re of S
+            i.append(float(c))  # Im of S
         return f, r, i, 'data parsed'
 
+    # make accessible specific range of numerical data choosen with interactive plot 
+    global interval_range, interval_start, interval_end
+
+    data = []
+    data_format_snp = False
+    uploaded_file = st.file_uploader('Upload a csv')
+    if uploaded_file is not None:
+        data = uploaded_file.readlines()
+        if uploaded_file.name[-4:-2]=='.s' and uploaded_file.name[-1]== 'p':
+            data_format_snp = True
+
     validator_status = '...'
-    # calculate
+    ace_preview_markers = []
+
+    # data loaded
     circle_params = []
     if len(data) > 0:
-        f, r, i, validator_status = unpack_data(data)
-        plot_interact_abs_from_f(f, r, i)
-
-        f_cut=f[interval_start:interval_end]
-        r_cut=r[interval_start:interval_end]
-        i_cut=i[interval_start:interval_end]
-
-        if validator_status == 'data parsed':
-            Q0, sigmaQ0, QL, sigmaQl, circle_params = calc_function(f_cut, r_cut, i_cut, select_coupling_losses)
-            # Q0 = round_up(Q0)
-            # sigmaQ0 = round_up(sigmaQ0)
-            # QL = round_up(QL)
-            # sigmaQl = round_up(sigmaQl)
-            if select_coupling_losses:
-                st.write("Lossy coupling")
-            else:
-                st.write("Cable attenuation")
 
-            out_precision='0.7f'
-            st.latex(r'Q_0 =' + f'{format(Q0, out_precision)} \pm {format(sigmaQ0, out_precision)},  ' + 
-                r'\;\;\varepsilon_{Q_0} =' + f'{format(sigmaQ0 / Q0, out_precision)}')
-            st.latex(r'Q_L =' + f'{format(QL, out_precision)} \pm {format(sigmaQl, out_precision)},  ' + 
-                r'\;\;\varepsilon_{Q_L} =' + f'{format(sigmaQl / QL, out_precision)}')
+        validator_status = read_data(data)
+        if validator_status == 'data read: success':
+            hz, select_measurement_parameter, select_data_representation, input_ref_resistance=parse_heading(data)
+
+            col1, col2 = st.columns(2)
+            select_measurement_parameter = col1.selectbox('Measurement parameter',
+                                                      ['S', 'Z'],
+                                                      select_measurement_parameter)
+            select_data_representation = col1.selectbox('Data representation',
+                                                    ['Frequency, real, imaginary',
+                                                     'Frequency, magnitude, angle'],
+                                                     select_data_representation)
+            if select_measurement_parameter=='Z':
+                input_ref_resistance = col1.number_input(
+                    "Reference resistance:", min_value=0, value=input_ref_resistance)                                    
+            input_start_line = col1.number_input(
+                "First line of data:", min_value=1, max_value=len(data))
+            input_end_line = col1.number_input(
+                "Last line of data:", min_value=1, max_value=len(data), value=len(data))
+
+            f, r, i, validator_status = unpack_data(data, input_start_line, input_end_line, hz)
+            # Ace editor to show choosen data columns and rows
+            with col2.expander("File preview"):
+                # web development is fundamentally imposible without such hacks
+                # if we have so little 'official' functionality in libs and this lack of documentation
+
+                # yellow ~ ace_step
+                # light yellow ~ ace_highlight-marker
+                # green ~ ace_stack
+                # red ~ ace_error-marker
+
+                # st.markdown('''<style>
+                # .choosen_option_1
+                # {
+                # color: rgb(49, 51, 63);
+                # }</style>''', unsafe_allow_html=True)
+
+                # markdown injection does not work, since ace is in a different .html accessible via iframe
+                # markers format:
+                #[{"startRow": 2,"startCol": 0,"endRow": 2,"endCol": 3,"className": "ace_error-marker","type": "text"}]
+                ace_preview_markers.append(
+                    {"startRow": input_start_line,"startCol": 0,
+                    "endRow": input_end_line+1,"endCol": 0,"className": "ace_highlight-marker","type": "text"})
+                text_value = "Frequency,Hz  | Re(S11) | Im(S11)\n" + \
+                    ''.join(data).strip()
+                st_ace(value=text_value,
+                       readonly=True,
+                       auto_update=True,
+                       placeholder="Your data is empty",
+                       markers=ace_preview_markers,
+                       height="300px")
+
+            st.write("Use range slider to choose best suitable data interval")
+            plot_interact_abs_from_f(f, r, i)
+
+            select_coupling_losses = st.checkbox(
+                'Apply corrections for coupling losses (lossy coupling)')
+            f_cut, r_cut, i_cut = (x[interval_start:interval_end]
+                                   for x in (f, r, i))
+
+            if validator_status == 'data parsed':
+                Q0, sigmaQ0, QL, sigmaQl, circle_params = calc_function(
+                    f_cut, r_cut, i_cut, select_coupling_losses)
+                # Q0 = round_up(Q0)
+                # sigmaQ0 = round_up(sigmaQ0)
+                # QL = round_up(QL)
+                # sigmaQl = round_up(sigmaQl)
+                if select_coupling_losses:
+                    st.write("Lossy coupling")
+                else:
+                    st.write("Cable attenuation")
+
+                out_precision = '0.7f'
+                st.latex(r'Q_0 =' + f'{format(Q0, out_precision)} \pm {format(sigmaQ0, out_precision)},  ' +
+                         r'\;\;\varepsilon_{Q_0} =' + f'{format(sigmaQ0 / Q0, out_precision)}')
+                st.latex(r'Q_L =' + f'{format(QL, out_precision)} \pm {format(sigmaQl, out_precision)},  ' +
+                         r'\;\;\varepsilon_{Q_L} =' + f'{format(sigmaQl / QL, out_precision)}')
 
     st.write("Status: " + validator_status)
 
-    if len(data) > 0:
-        f, r, i, validator_status = unpack_data(data)
-        if validator_status == 'data parsed':
+    if len(data) > 0 and validator_status == 'data parsed':
+        with st.expander("Show static abs(S) plot"):
             plot_ref_from_f(f_cut, r_cut, i_cut)
-            plot_data(r_cut, i_cut, circle_params)
+        plot_data(r_cut, i_cut, circle_params)