Coverage for src/ensae_teaching_cs/automation_students/projects_helper.py: 13%

1""" 

2@file 

3@brief A couple of functons which automates everything. 

4""" 

5 

6import os 

7import pandas 

8from pyquickhelper.loghelper import fLOG 

9from pyquickhelper.filehelper import encrypt_stream 

10from pymmails import MailBoxImap, EmailMessageRenderer, EmailMessageListRenderer 

11from pymmails.render.email_message_style import template_email_html_short 

12from .projects_repository import ProjectsRepository 

13from .mail_helper import grab_addresses 

14 

15 

16def extract_students_mails_from_gmail_and_stores_in_folders(folder=".", filemails="emails.txt", 

17 user=None, pwd=None, server="imap.gmail.com", 

18 mailfolder=[ 

19 "ensae/ENSAE_2016_3A"], 

20 date="1-Jan-2016", zipfilename="projet_3A_2016.zip", 

21 zipencpwd=b"sixteenbyteskeys", dataframe=None, 

22 columns={ 

23 "name": "nom_prenom", "group": "groupe", "subject": "sujet"}, 

24 skip_names=None, process_name=None, 

25 title="List of emails", nolink_if=None, fLOG=fLOG): 

26 """ 

27 The scenario is the following: 

28 

29 * You are the teacher. 

30 * Students started their projects at date *t*. 

31 * They can work alone or by group. 

32 * They send mails, you reply. 

33 * Their address mail follows the convention: ``<first name>.<last name>@anything`` 

34 so it is to associate a mail address to a student name. 

35 * You move every mail you received in a separate folder in your inbox. 

36 * Sometime, you send a mail to everybody. 

37 * Finally they send their project with attachments. 

38 * You want to store everything (mails and attachements) in folders, one per group. 

39 * You want a summary of what was received. 

40 * You want to build a zip file to share their work with others teachers. 

41 * You want to update the folder if a new mail was sent. 

42 

43 This function looks into a folder of your inbox and grabs every mails and 

44 attachements from a groups of students. 

45 

46 @param folder where to store the results 

47 @param filemails files used to store students address, 

48 the operation is done once, remove the file 

49 to force the function to rebuild the information. 

50 @param user user of the gmail inbox 

51 @param pwd password of the gmail inbox 

52 @param server gmail server, it should be ``"imap.gmail.com"``, 

53 it works with others mail servers using the *IMAP* protocol 

54 @param mailfolder folder in your inbox to look into, 

55 there can be several 

56 @param date when to start looking (do not change the format, 

57 look at the default value) 

58 @param zipfilename name of the zip file to create 

59 @param zipencpwd the zip file is also encrypted for a safer share with this key 

60 and function `encrypt_stream <http://www.xavierdupre.fr/app/pyquickhelper/helpsphinx/ 

61 pyquickhelper/filehelper/encryption.html#pyquickhelper.filehelper.encryption.encrypt_stream>`_. 

62 @param dataframe dataframe which contains the definition of students groups 

63 @param columns columns the function will look into, students names, group definition 

64 (a unique number for all students in the same group), subject 

65 @param skip_names list of names to skip 

66 @param process_name to operate a transformation before matching students names with 

67 their emails 

68 @param title each group folder contains a html file connecting them, 

69 this is its title 

70 @param nolink_if The summary extracts links from url, it skips the urls which 

71 contains on the substrings included in that list (None to use a default set) 

72 @param fLOG logging function 

73 @return @see cl ProjectsRepository 

74 

75 By default, Gmail does not let you programmatically access you own inbox, 

76 you need to modify your gmail parameters to let this function do so. 

77 """ 

78 folder = os.path.abspath(".") 

79 filemails = os.path.join(folder, filemails) 

80 zipfilename = os.path.join(folder, zipfilename) 

81 zipfilenameenc = zipfilename + ".enc" 

82 

83 # load the groups 

84 if isinstance(dataframe, pandas.DataFrame): 

85 df = dataframe 

86 elif dataframe.endswith("xlsx"): 

87 fLOG("[extract_students_mails_from_gmail_and_stores_in_folders] read dataframe", dataframe) 

88 df = pandas.read_excel(dataframe, engine='openpyxl') 

89 else: 

90 df = pandas.read_csv(dataframe, sep="\t", encoding="utf8") 

91 

92 # check mails 

93 if "mail" not in columns: 

94 if os.path.exists(filemails): 

95 fLOG( 

96 "[extract_students_mails_from_gmail_and_stores_in_folders] read addresses from ", filemails) 

97 with open(filemails, "r", encoding="utf8") as f: 

98 lines = f.readlines() 

99 emails = [li.strip("\r\t\n ") for li in lines] 

100 else: 

101 fLOG( 

102 "[extract_students_mails_from_gmail_and_stores_in_folders] mine address ") 

103 box = MailBoxImap(user, pwd, server, ssl=True, fLOG=fLOG) 

104 box.login() 

105 emails = grab_addresses(box, mailfolder, date, fLOG=fLOG) 

106 box.logout() 

107 

108 with open(filemails, "w", encoding="utf8") as f: 

109 f.write("\n".join(emails)) 

110 else: 

111 # nothing to do mail already present 

112 emails = set(df[columns["mail"]]) 

113 

114 # we remove empty names 

115 df = df[~df[columns["name"]].isnull()].copy() 

116 

117 if process_name: 

118 df[columns["name"]] = df[columns["name"]].apply( 

119 lambda f: process_name(f)) 

120 

121 fLOG("[extract_students_mails_from_gmail_and_stores_in_folders] create groups folders in", folder) 

122 proj = ProjectsRepository(folder, fLOG=fLOG) 

123 

124 proj = ProjectsRepository.create_folders_from_dataframe(df, folder, 

125 col_subject=columns[ 

126 "subject"], fLOG=fLOG, col_group=columns["group"], 

127 col_student=columns[ 

128 "name"], email_function=emails, skip_if_nomail=False, 

129 col_mail=columns["mail"], must_have_email=True, skip_names=skip_names) 

130 fLOG("[extract_students_mails_from_gmail_and_stores_in_folders] nb groups", len( 

131 proj.Groups)) 

132 

133 # gathers mails 

134 email_renderer = EmailMessageRenderer(tmpl=template_email_html_short, 

135 fLOG=fLOG) 

136 renderer = EmailMessageListRenderer(title=title, email_renderer=email_renderer, 

137 fLOG=fLOG) 

138 

139 box = MailBoxImap(user, pwd, server, ssl=True, fLOG=fLOG) 

140 box.login() 

141 proj.dump_group_mails(renderer, group=None, mailbox=box, subfolder=mailfolder, 

142 date=date, overwrite=False, skip_if_empty=True) 

143 

144 box.logout() 

145 

146 # cleaning files 

147 for group in proj.Groups: 

148 files = list(proj.enumerate_group_files(group)) 

149 att = [_ for _ in files if ".html" in _] 

150 if len(att) <= 1: 

151 fLOG( 

152 f"[extract_students_mails_from_gmail_and_stores_in_folders] remove '{group}'") 

153 proj.remove_group(group) 

154 

155 # unzip files and convert notebooks 

156 for group in proj.Groups: 

157 proj.unzip_convert(group) 

158 

159 summary = os.path.join(folder, "index.html") 

160 fLOG( 

161 f"[extract_students_mails_from_gmail_and_stores_in_folders] write summary '{summary}'") 

162 if os.path.exists(summary): 

163 os.remove(summary) 

164 proj.write_run_command() 

165 proj.write_summary(nolink_if=nolink_if) 

166 

167 fLOG("[extract_students_mails_from_gmail_and_stores_in_folders] zip everything in", zipfilename) 

168 if os.path.exists(zipfilename): 

169 os.remove(zipfilename) 

170 proj.zip_group(None, zipfilename, 

171 addition=["index.html", "mail_style.css", "emails.txt"]) 

172 

173 fLOG("[extract_students_mails_from_gmail_and_stores_in_folders] encrypt the zip file in '{}'.".format( 

174 zipfilenameenc)) 

175 if os.path.exists(zipfilenameenc): 

176 os.remove(zipfilenameenc) 

177 encrypt_stream(zipencpwd, zipfilename, zipfilenameenc, chunksize=2 ** 30) 

178 

179 return proj